1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Thanks to
10 * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
11 * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
12 * Denes
13 * Chris Clark
14 * clach05
15 * Denes Lengyel
16 * and many others who have contributed to current and previous versions
17
18 This file contains the DAL support for many relational databases,
19 including:
20 - SQLite & SpatiaLite
21 - MySQL
22 - Postgres
23 - Firebird
24 - Oracle
25 - MS SQL
26 - DB2
27 - Interbase
28 - Ingres
29 - Informix (9+ and SE)
30 - SapDB (experimental)
31 - Cubrid (experimental)
32 - CouchDB (experimental)
33 - MongoDB (in progress)
34 - Google:nosql
35 - Google:sql
36 - Teradata
37 - IMAP (experimental)
38
39 Example of usage:
40
41 >>> # from dal import DAL, Field
42
43 ### create DAL connection (and create DB if it doesn't exist)
44 >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
45 ... folder=None)
46
47 ### define a table 'person' (create/alter as necessary)
48 >>> person = db.define_table('person',Field('name','string'))
49
50 ### insert a record
51 >>> id = person.insert(name='James')
52
53 ### retrieve it by id
54 >>> james = person(id)
55
56 ### retrieve it by name
57 >>> james = person(name='James')
58
59 ### retrieve it by arbitrary query
60 >>> query = (person.name=='James') & (person.name.startswith('J'))
61 >>> james = db(query).select(person.ALL)[0]
62
63 ### update one record
64 >>> james.update_record(name='Jim')
65 <Row {'id': 1, 'name': 'Jim'}>
66
67 ### update multiple records by query
68 >>> db(person.name.like('J%')).update(name='James')
69 1
70
71 ### delete records by query
72 >>> db(person.name.lower() == 'jim').delete()
73 0
74
75 ### retrieve multiple records (rows)
76 >>> people = db(person).select(orderby=person.name,
77 ... groupby=person.name, limitby=(0,100))
78
79 ### further filter them
80 >>> james = people.find(lambda row: row.name == 'James').first()
81 >>> print james.id, james.name
82 1 James
83
84 ### check aggregates
85 >>> counter = person.id.count()
86 >>> print db(person).select(counter).first()(counter)
87 1
88
89 ### delete one record
90 >>> james.delete_record()
91 1
92
93 ### delete (drop) entire database table
94 >>> person.drop()
95
96 Supported field types:
97 id string text boolean integer double decimal password upload
98 blob time date datetime
99
100 Supported DAL URI strings:
101 'sqlite://test.db'
102 'spatialite://test.db'
103 'sqlite:memory'
104 'spatialite:memory'
105 'jdbc:sqlite://test.db'
106 'mysql://root:none@localhost/test'
107 'postgres://mdipierro:password@localhost/test'
108 'postgres:psycopg2://mdipierro:password@localhost/test'
109 'postgres:pg8000://mdipierro:password@localhost/test'
110 'jdbc:postgres://mdipierro:none@localhost/test'
111 'mssql://web2py:none@A64X2/web2py_test'
112 'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
113 'oracle://username:password@database'
114 'firebird://user:password@server:3050/database'
115 'db2://DSN=dsn;UID=user;PWD=pass'
116 'firebird://username:password@hostname/database'
117 'firebird_embedded://username:password@c://path'
118 'informix://user:password@server:3050/database'
119 'informixu://user:password@server:3050/database' # unicode informix
120 'ingres://database' # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name'
121 'google:datastore' # for google app engine datastore
122 'google:sql' # for google app engine with sql (mysql compatible)
123 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
124 'imap://user:password@server:port' # experimental
125 'mongodb://user:password@server:port/database' # experimental
126
127 For more info:
128 help(DAL)
129 help(Field)
130 """
131
132
133
134
135
136 __all__ = ['DAL', 'Field']
137
138 DEFAULTLENGTH = {'string':512,
139 'password':512,
140 'upload':512,
141 'text':2**15,
142 'blob':2**31}
143 TIMINGSSIZE = 100
144 SPATIALLIBS = {
145 'Windows':'libspatialite',
146 'Linux':'libspatialite.so',
147 'Darwin':'libspatialite.dylib'
148 }
149 DEFAULT_URI = 'sqlite://dummy.db'
150
151 import re
152 import sys
153 import locale
154 import os
155 import types
156 import datetime
157 import threading
158 import time
159 import csv
160 import cgi
161 import copy
162 import socket
163 import logging
164 import base64
165 import shutil
166 import marshal
167 import decimal
168 import struct
169 import urllib
170 import hashlib
171 import uuid
172 import glob
173 import traceback
174 import platform
175
176 PYTHON_VERSION = sys.version_info[0]
177 if PYTHON_VERSION == 2:
178 import cPickle as pickle
179 import cStringIO as StringIO
180 import copy_reg as copyreg
181 hashlib_md5 = hashlib.md5
182 bytes, unicode = str, unicode
183 else:
184 import pickle
185 from io import StringIO as StringIO
186 import copyreg
187 long = int
188 hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
189 bytes, unicode = bytes, str
190
191 CALLABLETYPES = (types.LambdaType, types.FunctionType,
192 types.BuiltinFunctionType,
193 types.MethodType, types.BuiltinMethodType)
194
195 TABLE_ARGS = set(
196 ('migrate','primarykey','fake_migrate','format','redefine',
197 'singular','plural','trigger_name','sequence_name','fields',
198 'common_filter','polymodel','table_class','on_define','actual_name'))
199
200 SELECT_ARGS = set(
201 ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
202 'distinct', 'having', 'join','for_update', 'processor','cacheable', 'orderby_on_limitby'))
203
204 ogetattr = object.__getattribute__
205 osetattr = object.__setattr__
206 exists = os.path.exists
207 pjoin = os.path.join
208
209
210
211
212 try:
213 from gluon.utils import web2py_uuid
214 except (ImportError, SystemError):
215 import uuid
217
218 try:
219 import portalocker
220 have_portalocker = True
221 except ImportError:
222 have_portalocker = False
223
224 try:
225 from gluon import serializers
226 have_serializers = True
227 except ImportError:
228 have_serializers = False
229 try:
230 import json as simplejson
231 except ImportError:
232 try:
233 import gluon.contrib.simplejson as simplejson
234 except ImportError:
235 simplejson = None
236
237 LOGGER = logging.getLogger("web2py.dal")
238 DEFAULT = lambda:0
239
240 GLOBAL_LOCKER = threading.RLock()
241 THREAD_LOCAL = threading.local()
242
243
244
245
246 REGEX_TYPE = re.compile('^([\w\_\:]+)')
247 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
248 REGEX_W = re.compile('^\w+$')
249 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$')
250 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)(\.(?P<name>\w+))?\.\w+$')
251 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
252 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
253 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
254 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
255 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
256 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
257 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
258 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
259 REGEX_QUOTES = re.compile("'[^']*'")
260 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
261 REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
262 REGEX_NOPASSWD = re.compile('\/\/[\w\.\-]+[\:\/](.+)(?=@)')
263
264
265
266 DRIVERS = []
267
268 try:
269 from new import classobj
270 from google.appengine.ext import db as gae
271 from google.appengine.api import namespace_manager, rdbms
272 from google.appengine.api.datastore_types import Key
273 from google.appengine.ext.db.polymodel import PolyModel
274 DRIVERS.append('google')
275 except ImportError:
276 pass
277
278 if not 'google' in DRIVERS:
279
280 try:
281 from pysqlite2 import dbapi2 as sqlite2
282 DRIVERS.append('SQLite(sqlite2)')
283 except ImportError:
284 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
285
286 try:
287 from sqlite3 import dbapi2 as sqlite3
288 DRIVERS.append('SQLite(sqlite3)')
289 except ImportError:
290 LOGGER.debug('no SQLite drivers sqlite3')
291
292 try:
293
294 try:
295 import gluon.contrib.pymysql as pymysql
296
297
298 pymysql.ESCAPE_REGEX = re.compile("'")
299 pymysql.ESCAPE_MAP = {"'": "''"}
300
301 except ImportError:
302 import pymysql
303 DRIVERS.append('MySQL(pymysql)')
304 except ImportError:
305 LOGGER.debug('no MySQL driver pymysql')
306
307 try:
308 import MySQLdb
309 DRIVERS.append('MySQL(MySQLdb)')
310 except ImportError:
311 LOGGER.debug('no MySQL driver MySQLDB')
312
313
314 try:
315 import psycopg2
316 from psycopg2.extensions import adapt as psycopg2_adapt
317 DRIVERS.append('PostgreSQL(psycopg2)')
318 except ImportError:
319 LOGGER.debug('no PostgreSQL driver psycopg2')
320
321 try:
322
323 try:
324 import gluon.contrib.pg8000.dbapi as pg8000
325 except ImportError:
326 import pg8000.dbapi as pg8000
327 DRIVERS.append('PostgreSQL(pg8000)')
328 except ImportError:
329 LOGGER.debug('no PostgreSQL driver pg8000')
330
331 try:
332 import cx_Oracle
333 DRIVERS.append('Oracle(cx_Oracle)')
334 except ImportError:
335 LOGGER.debug('no Oracle driver cx_Oracle')
336
337 try:
338 try:
339 import pyodbc
340 except ImportError:
341 try:
342 import gluon.contrib.pypyodbc as pyodbc
343 except Exception, e:
344 raise ImportError(str(e))
345 DRIVERS.append('MSSQL(pyodbc)')
346 DRIVERS.append('DB2(pyodbc)')
347 DRIVERS.append('Teradata(pyodbc)')
348 DRIVERS.append('Ingres(pyodbc)')
349 except ImportError:
350 LOGGER.debug('no MSSQL/DB2/Teradata/Ingres driver pyodbc')
351
352 try:
353 import Sybase
354 DRIVERS.append('Sybase(Sybase)')
355 except ImportError:
356 LOGGER.debug('no Sybase driver')
357
358 try:
359 import kinterbasdb
360 DRIVERS.append('Interbase(kinterbasdb)')
361 DRIVERS.append('Firebird(kinterbasdb)')
362 except ImportError:
363 LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
364
365 try:
366 import fdb
367 DRIVERS.append('Firebird(fdb)')
368 except ImportError:
369 LOGGER.debug('no Firebird driver fdb')
370
371 try:
372 import firebirdsql
373 DRIVERS.append('Firebird(firebirdsql)')
374 except ImportError:
375 LOGGER.debug('no Firebird driver firebirdsql')
376
377 try:
378 import informixdb
379 DRIVERS.append('Informix(informixdb)')
380 LOGGER.warning('Informix support is experimental')
381 except ImportError:
382 LOGGER.debug('no Informix driver informixdb')
383
384 try:
385 import sapdb
386 DRIVERS.append('SQL(sapdb)')
387 LOGGER.warning('SAPDB support is experimental')
388 except ImportError:
389 LOGGER.debug('no SAP driver sapdb')
390
391 try:
392 import cubriddb
393 DRIVERS.append('Cubrid(cubriddb)')
394 LOGGER.warning('Cubrid support is experimental')
395 except ImportError:
396 LOGGER.debug('no Cubrid driver cubriddb')
397
398 try:
399 from com.ziclix.python.sql import zxJDBC
400 import java.sql
401
402 from org.sqlite import JDBC
403 zxJDBC_sqlite = java.sql.DriverManager
404 DRIVERS.append('PostgreSQL(zxJDBC)')
405 DRIVERS.append('SQLite(zxJDBC)')
406 LOGGER.warning('zxJDBC support is experimental')
407 is_jdbc = True
408 except ImportError:
409 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
410 is_jdbc = False
411
412 try:
413 import couchdb
414 DRIVERS.append('CouchDB(couchdb)')
415 except ImportError:
416 LOGGER.debug('no Couchdb driver couchdb')
417
418 try:
419 import pymongo
420 DRIVERS.append('MongoDB(pymongo)')
421 except:
422 LOGGER.debug('no MongoDB driver pymongo')
423
424 try:
425 import imaplib
426 DRIVERS.append('IMAP(imaplib)')
427 except:
428 LOGGER.debug('no IMAP driver imaplib')
429
430 PLURALIZE_RULES = [
431 (re.compile('child$'), re.compile('child$'), 'children'),
432 (re.compile('oot$'), re.compile('oot$'), 'eet'),
433 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
434 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
435 (re.compile('sis$'), re.compile('sis$'), 'ses'),
436 (re.compile('man$'), re.compile('man$'), 'men'),
437 (re.compile('ife$'), re.compile('ife$'), 'ives'),
438 (re.compile('eau$'), re.compile('eau$'), 'eaux'),
439 (re.compile('lf$'), re.compile('lf$'), 'lves'),
440 (re.compile('[sxz]$'), re.compile('$'), 'es'),
441 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
442 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
443 (re.compile('$'), re.compile('$'), 's'),
444 ]
451
456
459
462
464
467
469 regex = re.compile('\.keyword(?=\w)')
470 a = regex.sub('."%s"' % keyword,a)
471 return a
472
473 if 'google' in DRIVERS:
474
475 is_jdbc = False
478 """
479 GAE decimal implementation
480 """
481 data_type = decimal.Decimal
482
483 - def __init__(self, precision, scale, **kwargs):
484 super(GAEDecimalProperty, self).__init__(self, **kwargs)
485 d = '1.'
486 for x in range(scale):
487 d += '0'
488 self.round = decimal.Decimal(d)
489
497
499 if value is None or value == '':
500 return None
501 else:
502 return decimal.Decimal(value).quantize(self.round)
503
505 value = super(GAEDecimalProperty, self).validate(value)
506 if value is None or isinstance(value, decimal.Decimal):
507 return value
508 elif isinstance(value, basestring):
509 return decimal.Decimal(value)
510 raise gae.BadValueError("Property %s must be a Decimal or string."\
511 % self.name)
512
518
519 POOLS = {}
520 check_active_connection = True
521
522 @staticmethod
525
526
527
528 - def close(self,action='commit',really=True):
545
546 @staticmethod
548 """ to close cleanly databases in a multithreaded environment """
549 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
550 for db_uid, db_group in dbs:
551 for db in db_group:
552 if hasattr(db,'_adapter'):
553 db._adapter.close(action)
554 getattr(THREAD_LOCAL,'db_instances',{}).clear()
555 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
556 if callable(action):
557 action(None)
558 return
559
572
574 """hook for the after_connection parameter"""
575 if callable(self._after_connection):
576 self._after_connection(self)
577 self.after_connection()
578
580 """ this it is supposed to be overloaded by adapters"""
581 pass
582
584 """
585 this function defines: self.connection and self.cursor
586 (iff cursor is True)
587 if self.pool_size>0 it will try pull the connection from the pool
588 if the connection is not active (closed by db server) it will loop
589 if not self.pool_size or no active connections in pool makes a new one
590 """
591 if getattr(self,'connection', None) != None:
592 return
593 if f is None:
594 f = self.connector
595
596
597
598
599
600 if not self.pool_size:
601 self.connection = f()
602 self.cursor = cursor and self.connection.cursor()
603 else:
604 uri = self.uri
605 POOLS = ConnectionPool.POOLS
606 while True:
607 GLOBAL_LOCKER.acquire()
608 if not uri in POOLS:
609 POOLS[uri] = []
610 if POOLS[uri]:
611 self.connection = POOLS[uri].pop()
612 GLOBAL_LOCKER.release()
613 self.cursor = cursor and self.connection.cursor()
614 try:
615 if self.cursor and self.check_active_connection:
616 self.execute('SELECT 1;')
617 break
618 except:
619 pass
620 else:
621 GLOBAL_LOCKER.release()
622 self.connection = f()
623 self.cursor = cursor and self.connection.cursor()
624 break
625 self.after_connection_hook()
626
633 native_json = False
634 driver = None
635 driver_name = None
636 drivers = ()
637 connection = None
638 commit_on_alter_table = False
639 support_distributed_transaction = False
640 uploads_in_blob = False
641 can_select_for_update = True
642 dbpath = None
643 folder = None
644
645 TRUE = 'T'
646 FALSE = 'F'
647 T_SEP = ' '
648 QUOTE_TEMPLATE = '"%s"'
649
650 types = {
651 'boolean': 'CHAR(1)',
652 'string': 'CHAR(%(length)s)',
653 'text': 'TEXT',
654 'json': 'TEXT',
655 'password': 'CHAR(%(length)s)',
656 'blob': 'BLOB',
657 'upload': 'CHAR(%(length)s)',
658 'integer': 'INTEGER',
659 'bigint': 'INTEGER',
660 'float':'DOUBLE',
661 'double': 'DOUBLE',
662 'decimal': 'DOUBLE',
663 'date': 'DATE',
664 'time': 'TIME',
665 'datetime': 'TIMESTAMP',
666 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
667 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
668 'list:integer': 'TEXT',
669 'list:string': 'TEXT',
670 'list:reference': 'TEXT',
671
672 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT',
673 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
674 }
675
677 if not hasattr(self.driver, "OperationalError"):
678 return None
679 return isinstance(exception, self.driver.OperationalError)
680
682 if not hasattr(self.driver, "ProgrammingError"):
683 return None
684 return isinstance(exception, self.driver.ProgrammingError)
685
687 pkeys = getattr(table,'_primarykey',None)
688 if pkeys:
689 return table[pkeys[0]] != None
690 else:
691 return table._id != None
692
694 return "'%s'" % obj.replace("'", "''")
695
697 if isinstance(obj,(int,float)):
698 return str(obj)
699 return self.adapt(str(obj))
700
702 """
703 to be used ONLY for files that on GAE may not be on filesystem
704 """
705 return exists(filename)
706
707 - def file_open(self, filename, mode='rb', lock=True):
708 """
709 to be used ONLY for files that on GAE may not be on filesystem
710 """
711 if have_portalocker and lock:
712 fileobj = portalocker.LockedFile(filename,mode)
713 else:
714 fileobj = open(filename,mode)
715 return fileobj
716
718 """
719 to be used ONLY for files that on GAE may not be on filesystem
720 """
721 if fileobj:
722 fileobj.close()
723
726
728 self.adapter_args = adapter_args
729 if getattr(self,'driver',None) != None:
730 return
731 drivers_available = [driver for driver in self.drivers
732 if driver in globals()]
733 if uri:
734 items = uri.split('://',1)[0].split(':')
735 request_driver = items[1] if len(items)>1 else None
736 else:
737 request_driver = None
738 request_driver = request_driver or adapter_args.get('driver')
739 if request_driver:
740 if request_driver in drivers_available:
741 self.driver_name = request_driver
742 self.driver = globals().get(request_driver)
743 else:
744 raise RuntimeError("driver %s not available" % request_driver)
745 elif drivers_available:
746 self.driver_name = drivers_available[0]
747 self.driver = globals().get(self.driver_name)
748 else:
749 raise RuntimeError("no driver available %s" % str(self.drivers))
750
751 - def log(self, message, table=None):
752 """ Logs migrations
753
754 It will not log changes if logfile is not specified. Defaults
755 to sql.log
756 """
757
758 isabs = None
759 logfilename = self.adapter_args.get('logfile','sql.log')
760 writelog = bool(logfilename)
761 if writelog:
762 isabs = os.path.isabs(logfilename)
763
764 if table and table._dbt and writelog and self.folder:
765 if isabs:
766 table._loggername = logfilename
767 else:
768 table._loggername = pjoin(self.folder, logfilename)
769 logfile = self.file_open(table._loggername, 'a')
770 logfile.write(message)
771 self.file_close(logfile)
772
773
774 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
775 credential_decoder=IDENTITY, driver_args={},
776 adapter_args={},do_connect=True, after_connection=None):
777 self.db = db
778 self.dbengine = "None"
779 self.uri = uri
780 self.pool_size = pool_size
781 self.folder = folder
782 self.db_codec = db_codec
783 self._after_connection = after_connection
784 class Dummy(object):
785 lastrowid = 1
786 def __getattr__(self, value):
787 return lambda *a, **b: []
788 self.connection = Dummy()
789 self.cursor = Dummy()
790
792 return '%s_sequence' % tablename
793
795 return '%s_sequence' % tablename
796
799
800 - def create_table(self, table,
801 migrate=True,
802 fake_migrate=False,
803 polymodel=None):
804 db = table._db
805 fields = []
806
807 postcreation_fields = []
808 sql_fields = {}
809 sql_fields_aux = {}
810 TFK = {}
811 tablename = table._tablename
812 sortable = 0
813 types = self.types
814 for field in table:
815 sortable += 1
816 field_name = field.name
817 field_type = field.type
818 if isinstance(field_type,SQLCustomType):
819 ftype = field_type.native or field_type.type
820 elif field_type.startswith('reference'):
821 referenced = field_type[10:].strip()
822 if referenced == '.':
823 referenced = tablename
824 constraint_name = self.constraint_name(tablename, field_name)
825 if not '.' in referenced \
826 and referenced != tablename \
827 and hasattr(table,'_primarykey'):
828 ftype = types['integer']
829 else:
830 if hasattr(table,'_primarykey'):
831 rtablename,rfieldname = referenced.split('.')
832 rtable = db[rtablename]
833 rfield = rtable[rfieldname]
834
835 if rfieldname in rtable._primarykey or \
836 rfield.unique:
837 ftype = types[rfield.type[:9]] % \
838 dict(length=rfield.length)
839
840 if not rfield.unique and len(rtable._primarykey)>1:
841
842 if rtablename not in TFK:
843 TFK[rtablename] = {}
844 TFK[rtablename][rfieldname] = field_name
845 else:
846 ftype = ftype + \
847 types['reference FK'] % dict(
848 constraint_name = constraint_name,
849 foreign_key = '%s (%s)' % (rtablename,
850 rfieldname),
851 table_name = tablename,
852 field_name = field_name,
853 on_delete_action=field.ondelete)
854 else:
855
856 if referenced in db:
857 id_fieldname = db[referenced]._id.name
858 elif referenced == tablename:
859 id_fieldname = table._id.name
860 else:
861 id_fieldname = 'id'
862 ftype = types[field_type[:9]] % dict(
863 index_name = field_name+'__idx',
864 field_name = field_name,
865 constraint_name = constraint_name,
866 foreign_key = '%s (%s)' % (referenced,
867 id_fieldname),
868 on_delete_action=field.ondelete)
869 elif field_type.startswith('list:reference'):
870 ftype = types[field_type[:14]]
871 elif field_type.startswith('decimal'):
872 precision, scale = map(int,field_type[8:-1].split(','))
873 ftype = types[field_type[:7]] % \
874 dict(precision=precision,scale=scale)
875 elif field_type.startswith('geo'):
876 if not hasattr(self,'srid'):
877 raise RuntimeError('Adapter does not support geometry')
878 srid = self.srid
879 geotype, parms = field_type[:-1].split('(')
880 if not geotype in types:
881 raise SyntaxError(
882 'Field: unknown field type: %s for %s' \
883 % (field_type, field_name))
884 ftype = types[geotype]
885 if self.dbengine == 'postgres' and geotype == 'geometry':
886
887 dimension = 2
888 parms = parms.split(',')
889 if len(parms) == 3:
890 schema, srid, dimension = parms
891 elif len(parms) == 2:
892 schema, srid = parms
893 else:
894 schema = parms[0]
895 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
896 ftype = ftype % dict(schema=schema,
897 tablename=tablename,
898 fieldname=field_name, srid=srid,
899 dimension=dimension)
900 postcreation_fields.append(ftype)
901 elif not field_type in types:
902 raise SyntaxError('Field: unknown field type: %s for %s' % \
903 (field_type, field_name))
904 else:
905 ftype = types[field_type]\
906 % dict(length=field.length)
907 if not field_type.startswith('id') and \
908 not field_type.startswith('reference'):
909 if field.notnull:
910 ftype += ' NOT NULL'
911 else:
912 ftype += self.ALLOW_NULL()
913 if field.unique:
914 ftype += ' UNIQUE'
915 if field.custom_qualifier:
916 ftype += ' %s' % field.custom_qualifier
917
918
919 sql_fields[field_name] = dict(
920 length=field.length,
921 unique=field.unique,
922 notnull=field.notnull,
923 sortable=sortable,
924 type=str(field_type),
925 sql=ftype)
926
927 if field.notnull and not field.default is None:
928
929
930
931
932
933
934 not_null = self.NOT_NULL(field.default, field_type)
935 ftype = ftype.replace('NOT NULL', not_null)
936 sql_fields_aux[field_name] = dict(sql=ftype)
937
938
939 if not (self.dbengine == 'postgres' and \
940 field_type.startswith('geom')):
941 fields.append('%s %s' % (field_name, ftype))
942 other = ';'
943
944
945 if self.dbengine == 'mysql':
946 if not hasattr(table, "_primarykey"):
947 fields.append('PRIMARY KEY(%s)' % table._id.name)
948 other = ' ENGINE=InnoDB CHARACTER SET utf8;'
949
950 fields = ',\n '.join(fields)
951 for rtablename in TFK:
952 rfields = TFK[rtablename]
953 pkeys = db[rtablename]._primarykey
954 fkeys = [ rfields[k] for k in pkeys ]
955 fields = fields + ',\n ' + \
956 types['reference TFK'] % dict(
957 table_name = tablename,
958 field_name=', '.join(fkeys),
959 foreign_table = rtablename,
960 foreign_key = ', '.join(pkeys),
961 on_delete_action = field.ondelete)
962
963 if getattr(table,'_primarykey',None):
964 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \
965 (tablename, fields,
966 self.PRIMARY_KEY(', '.join(table._primarykey)),other)
967 else:
968 query = "CREATE TABLE %s(\n %s\n)%s" % \
969 (tablename, fields, other)
970
971 if self.uri.startswith('sqlite:///') \
972 or self.uri.startswith('spatialite:///'):
973 path_encoding = sys.getfilesystemencoding() \
974 or locale.getdefaultlocale()[1] or 'utf8'
975 dbpath = self.uri[9:self.uri.rfind('/')]\
976 .decode('utf8').encode(path_encoding)
977 else:
978 dbpath = self.folder
979
980 if not migrate:
981 return query
982 elif self.uri.startswith('sqlite:memory')\
983 or self.uri.startswith('spatialite:memory'):
984 table._dbt = None
985 elif isinstance(migrate, str):
986 table._dbt = pjoin(dbpath, migrate)
987 else:
988 table._dbt = pjoin(
989 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
990
991 if not table._dbt or not self.file_exists(table._dbt):
992 if table._dbt:
993 self.log('timestamp: %s\n%s\n'
994 % (datetime.datetime.today().isoformat(),
995 query), table)
996 if not fake_migrate:
997 self.create_sequence_and_triggers(query,table)
998 table._db.commit()
999
1000
1001 for query in postcreation_fields:
1002 self.execute(query)
1003 table._db.commit()
1004 if table._dbt:
1005 tfile = self.file_open(table._dbt, 'w')
1006 pickle.dump(sql_fields, tfile)
1007 self.file_close(tfile)
1008 if fake_migrate:
1009 self.log('faked!\n', table)
1010 else:
1011 self.log('success!\n', table)
1012 else:
1013 tfile = self.file_open(table._dbt, 'r')
1014 try:
1015 sql_fields_old = pickle.load(tfile)
1016 except EOFError:
1017 self.file_close(tfile)
1018 raise RuntimeError('File %s appears corrupted' % table._dbt)
1019 self.file_close(tfile)
1020 if sql_fields != sql_fields_old:
1021 self.migrate_table(table,
1022 sql_fields, sql_fields_old,
1023 sql_fields_aux, None,
1024 fake_migrate=fake_migrate)
1025 return query
1026
1027 - def migrate_table(
1028 self,
1029 table,
1030 sql_fields,
1031 sql_fields_old,
1032 sql_fields_aux,
1033 logfile,
1034 fake_migrate=False,
1035 ):
1036
1037
1038 db = table._db
1039 db._migrated.append(table._tablename)
1040 tablename = table._tablename
1041 def fix(item):
1042 k,v=item
1043 if not isinstance(v,dict):
1044 v=dict(type='unknown',sql=v)
1045 return k.lower(),v
1046
1047
1048 sql_fields = dict(map(fix,sql_fields.iteritems()))
1049 sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
1050 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
1051 if db._debug:
1052 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields))
1053
1054 keys = sql_fields.keys()
1055 for key in sql_fields_old:
1056 if not key in keys:
1057 keys.append(key)
1058 new_add = self.concat_add(tablename)
1059
1060 metadata_change = False
1061 sql_fields_current = copy.copy(sql_fields_old)
1062 for key in keys:
1063 query = None
1064 if not key in sql_fields_old:
1065 sql_fields_current[key] = sql_fields[key]
1066 if self.dbengine in ('postgres',) and \
1067 sql_fields[key]['type'].startswith('geometry'):
1068
1069 query = [ sql_fields[key]['sql'] ]
1070 else:
1071 query = ['ALTER TABLE %s ADD %s %s;' % \
1072 (tablename, key,
1073 sql_fields_aux[key]['sql'].replace(', ', new_add))]
1074 metadata_change = True
1075 elif self.dbengine in ('sqlite', 'spatialite'):
1076 if key in sql_fields:
1077 sql_fields_current[key] = sql_fields[key]
1078 metadata_change = True
1079 elif not key in sql_fields:
1080 del sql_fields_current[key]
1081 ftype = sql_fields_old[key]['type']
1082 if (self.dbengine in ('postgres',) and
1083 ftype.startswith('geometry')):
1084 geotype, parms = ftype[:-1].split('(')
1085 schema = parms.split(',')[0]
1086 query = [ "SELECT DropGeometryColumn ('%(schema)s', "+
1087 "'%(table)s', '%(field)s');" %
1088 dict(schema=schema, table=tablename, field=key,) ]
1089 elif self.dbengine in ('firebird',):
1090 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
1091 else:
1092 query = ['ALTER TABLE %s DROP COLUMN %s;' %
1093 (tablename, key)]
1094 metadata_change = True
1095 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
1096 and not (key in table.fields and
1097 isinstance(table[key].type, SQLCustomType)) \
1098 and not sql_fields[key]['type'].startswith('reference')\
1099 and not sql_fields[key]['type'].startswith('double')\
1100 and not sql_fields[key]['type'].startswith('id'):
1101 sql_fields_current[key] = sql_fields[key]
1102 t = tablename
1103 tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
1104 if self.dbengine in ('firebird',):
1105 drop_expr = 'ALTER TABLE %s DROP %s;'
1106 else:
1107 drop_expr = 'ALTER TABLE %s DROP COLUMN %s;'
1108 key_tmp = key + '__tmp'
1109 query = ['ALTER TABLE %s ADD %s %s;' % (t, key_tmp, tt),
1110 'UPDATE %s SET %s=%s;' % (t, key_tmp, key),
1111 drop_expr % (t, key),
1112 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1113 'UPDATE %s SET %s=%s;' % (t, key, key_tmp),
1114 drop_expr % (t, key_tmp)]
1115 metadata_change = True
1116 elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
1117 sql_fields_current[key] = sql_fields[key]
1118 metadata_change = True
1119
1120 if query:
1121 self.log('timestamp: %s\n'
1122 % datetime.datetime.today().isoformat(), table)
1123 db['_lastsql'] = '\n'.join(query)
1124 for sub_query in query:
1125 self.log(sub_query + '\n', table)
1126 if fake_migrate:
1127 if db._adapter.commit_on_alter_table:
1128 self.save_dbt(table,sql_fields_current)
1129 self.log('faked!\n', table)
1130 else:
1131 self.execute(sub_query)
1132
1133
1134
1135
1136
1137 if db._adapter.commit_on_alter_table:
1138 db.commit()
1139 self.save_dbt(table,sql_fields_current)
1140 self.log('success!\n', table)
1141
1142 elif metadata_change:
1143 self.save_dbt(table,sql_fields_current)
1144
1145 if metadata_change and not (query and db._adapter.commit_on_alter_table):
1146 db.commit()
1147 self.save_dbt(table,sql_fields_current)
1148 self.log('success!\n', table)
1149
1150 - def save_dbt(self,table, sql_fields_current):
1151 tfile = self.file_open(table._dbt, 'w')
1152 pickle.dump(sql_fields_current, tfile)
1153 self.file_close(tfile)
1154
1155 - def LOWER(self, first):
1157
1158 - def UPPER(self, first):
1160
1161 - def COUNT(self, first, distinct=None):
1162 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \
1163 % self.expand(first)
1164
1166 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1167
1168 - def EPOCH(self, first):
1170
1173
1176
1179
1182
1185
1186 - def NOT_NULL(self, default, field_type):
1187 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1188
1190 expressions = [self.expand(first)]+[self.expand(e) for e in second]
1191 return 'COALESCE(%s)' % ','.join(expressions)
1192
1195
1196 - def RAW(self, first):
1198
1201
1203 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1204
1206 return 'PRIMARY KEY(%s)' % key
1207
1208 - def _drop(self, table, mode):
1209 return ['DROP TABLE %s;' % table]
1210
1211 - def drop(self, table, mode=''):
1225
1226 - def _insert(self, table, fields):
1233
1235 return 'INSERT INTO %s DEFAULT VALUES;' % table
1236
1237 - def insert(self, table, fields):
1238 query = self._insert(table,fields)
1239 try:
1240 self.execute(query)
1241 except Exception:
1242 e = sys.exc_info()[1]
1243 if hasattr(table,'_on_insert_error'):
1244 return table._on_insert_error(table,fields,e)
1245 raise e
1246 if hasattr(table,'_primarykey'):
1247 return dict([(k[0].name, k[1]) for k in fields \
1248 if k[0].name in table._primarykey])
1249 id = self.lastrowid(table)
1250 if not isinstance(id,int):
1251 return id
1252 rid = Reference(id)
1253 (rid._table, rid._record) = (table, None)
1254 return rid
1255
1257 return [self.insert(table,item) for item in items]
1258
1259 - def NOT(self, first):
1261
1262 - def AND(self, first, second):
1264
1265 - def OR(self, first, second):
1267
1268 - def BELONGS(self, first, second):
1269 if isinstance(second, str):
1270 return '(%s IN (%s))' % (self.expand(first), second[:-1])
1271 if not second:
1272 return '(1=0)'
1273 items = ','.join(self.expand(item, first.type) for item in second)
1274 return '(%s IN (%s))' % (self.expand(first), items)
1275
1276 - def REGEXP(self, first, second):
1277 "regular expression operator"
1278 raise NotImplementedError
1279
1280 - def LIKE(self, first, second):
1281 "case sensitive like operator"
1282 raise NotImplementedError
1283
1284 - def ILIKE(self, first, second):
1285 "case in-sensitive like operator"
1286 return '(%s LIKE %s)' % (self.expand(first),
1287 self.expand(second, 'string'))
1288
1290 return '(%s LIKE %s)' % (self.expand(first),
1291 self.expand(second+'%', 'string'))
1292
1294 return '(%s LIKE %s)' % (self.expand(first),
1295 self.expand('%'+second, 'string'))
1296
1297 - def CONTAINS(self,first,second,case_sensitive=False):
1298 if first.type in ('string','text', 'json'):
1299 if isinstance(second,Expression):
1300 second = Expression(None,self.CONCAT('%',Expression(
1301 None,self.REPLACE(second,('%','%%'))),'%'))
1302 else:
1303 second = '%'+str(second).replace('%','%%')+'%'
1304 elif first.type.startswith('list:'):
1305 if isinstance(second,Expression):
1306 second = Expression(None,self.CONCAT(
1307 '%|',Expression(None,self.REPLACE(
1308 Expression(None,self.REPLACE(
1309 second,('%','%%'))),('|','||'))),'|%'))
1310 else:
1311 second = '%|'+str(second).replace('%','%%')\
1312 .replace('|','||')+'|%'
1313 op = case_sensitive and self.LIKE or self.ILIKE
1314 return op(first,second)
1315
1316 - def EQ(self, first, second=None):
1321
1322 - def NE(self, first, second=None):
1327
1328 - def LT(self,first,second=None):
1329 if second is None:
1330 raise RuntimeError("Cannot compare %s < None" % first)
1331 return '(%s < %s)' % (self.expand(first),
1332 self.expand(second,first.type))
1333
1334 - def LE(self,first,second=None):
1335 if second is None:
1336 raise RuntimeError("Cannot compare %s <= None" % first)
1337 return '(%s <= %s)' % (self.expand(first),
1338 self.expand(second,first.type))
1339
1340 - def GT(self,first,second=None):
1341 if second is None:
1342 raise RuntimeError("Cannot compare %s > None" % first)
1343 return '(%s > %s)' % (self.expand(first),
1344 self.expand(second,first.type))
1345
1346 - def GE(self,first,second=None):
1347 if second is None:
1348 raise RuntimeError("Cannot compare %s >= None" % first)
1349 return '(%s >= %s)' % (self.expand(first),
1350 self.expand(second,first.type))
1351
1353 return ftype in ('integer','boolean','double','bigint') or \
1354 ftype.startswith('decimal')
1355
1356 - def REPLACE(self, first, (second, third)):
1357 return 'REPLACE(%s,%s,%s)' % (self.expand(first,'string'),
1358 self.expand(second,'string'),
1359 self.expand(third,'string'))
1360
1363
1364 - def ADD(self, first, second):
1370
1371 - def SUB(self, first, second):
1374
1375 - def MUL(self, first, second):
1378
1379 - def DIV(self, first, second):
1382
1383 - def MOD(self, first, second):
1386
1387 - def AS(self, first, second):
1389
1390 - def ON(self, first, second):
1394
1397
1398 - def COMMA(self, first, second):
1400
1401 - def expand(self, expression, field_type=None):
1402 if isinstance(expression, Field):
1403 out = '%s.%s' % (expression.table._tablename, expression.name)
1404 if field_type == 'string' and not expression.type in (
1405 'string','text','json','password'):
1406 out = 'CAST(%s AS %s)' % (out, self.types['text'])
1407 return out
1408 elif isinstance(expression, (Expression, Query)):
1409 first = expression.first
1410 second = expression.second
1411 op = expression.op
1412 optional_args = expression.optional_args or {}
1413 if not second is None:
1414 out = op(first, second, **optional_args)
1415 elif not first is None:
1416 out = op(first,**optional_args)
1417 elif isinstance(op, str):
1418 if op.endswith(';'):
1419 op=op[:-1]
1420 out = '(%s)' % op
1421 else:
1422 out = op()
1423 return out
1424 elif field_type:
1425 return str(self.represent(expression,field_type))
1426 elif isinstance(expression,(list,tuple)):
1427 return ','.join(self.represent(item,field_type) \
1428 for item in expression)
1429 elif isinstance(expression, bool):
1430 return '1' if expression else '0'
1431 else:
1432 return str(expression)
1433
1436
1437 - def alias(self, table, alias):
1438 """
1439 Given a table object, makes a new table object
1440 with alias name.
1441 """
1442 other = copy.copy(table)
1443 other['_ot'] = other._ot or other._tablename
1444 other['ALL'] = SQLALL(other)
1445 other['_tablename'] = alias
1446 for fieldname in other.fields:
1447 other[fieldname] = copy.copy(other[fieldname])
1448 other[fieldname]._tablename = alias
1449 other[fieldname].tablename = alias
1450 other[fieldname].table = other
1451 table._db[alias] = other
1452 return other
1453
1455 tablename = table._tablename
1456 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1457
1459
1460 try:
1461 queries = table._db._adapter._truncate(table, mode)
1462 for query in queries:
1463 self.log(query + '\n', table)
1464 self.execute(query)
1465 table._db.commit()
1466 self.log('success!\n', table)
1467 finally:
1468 pass
1469
1470 - def _update(self, tablename, query, fields):
1471 if query:
1472 if use_common_filters(query):
1473 query = self.common_filter(query, [tablename])
1474 sql_w = ' WHERE ' + self.expand(query)
1475 else:
1476 sql_w = ''
1477 sql_v = ','.join(['%s=%s' % (field.name,
1478 self.expand(value, field.type)) \
1479 for (field, value) in fields])
1480 tablename = "%s" % self.db[tablename]
1481 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1482
1483 - def update(self, tablename, query, fields):
1484 sql = self._update(tablename, query, fields)
1485 try:
1486 self.execute(sql)
1487 except Exception:
1488 e = sys.exc_info()[1]
1489 table = self.db[tablename]
1490 if hasattr(table,'_on_update_error'):
1491 return table._on_update_error(table,query,fields,e)
1492 raise e
1493 try:
1494 return self.cursor.rowcount
1495 except:
1496 return None
1497
1498 - def _delete(self, tablename, query):
1499 if query:
1500 if use_common_filters(query):
1501 query = self.common_filter(query, [tablename])
1502 sql_w = ' WHERE ' + self.expand(query)
1503 else:
1504 sql_w = ''
1505 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1506
1507 - def delete(self, tablename, query):
1508 sql = self._delete(tablename, query)
1509
1510 db = self.db
1511 table = db[tablename]
1512 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by:
1513 deleted = [x[table._id.name] for x in db(query).select(table._id)]
1514
1515 self.execute(sql)
1516 try:
1517 counter = self.cursor.rowcount
1518 except:
1519 counter = None
1520
1521 if self.dbengine in ('sqlite', 'spatialite') and counter:
1522 for field in table._referenced_by:
1523 if field.type=='reference '+table._tablename \
1524 and field.ondelete=='CASCADE':
1525 db(field.belongs(deleted)).delete()
1526
1527 return counter
1528
1530 tablenames = self.tables(query)
1531 if len(tablenames)==1:
1532 return tablenames[0]
1533 elif len(tablenames)<1:
1534 raise RuntimeError("No table selected")
1535 else:
1536 raise RuntimeError("Too many tables selected")
1537
1539 db = self.db
1540 new_fields = []
1541 append = new_fields.append
1542 for item in fields:
1543 if isinstance(item,SQLALL):
1544 new_fields += item._table
1545 elif isinstance(item,str):
1546 if REGEX_TABLE_DOT_FIELD.match(item):
1547 tablename,fieldname = item.split('.')
1548 append(db[tablename][fieldname])
1549 else:
1550 append(Expression(db,lambda item=item:item))
1551 else:
1552 append(item)
1553
1554 if not new_fields:
1555 for table in tablenames:
1556 for field in db[table]:
1557 append(field)
1558 return new_fields
1559
1560 - def _select(self, query, fields, attributes):
1561 tables = self.tables
1562 for key in set(attributes.keys())-SELECT_ARGS:
1563 raise SyntaxError('invalid select attribute: %s' % key)
1564 args_get = attributes.get
1565 tablenames = tables(query)
1566 tablenames_for_common_filters = tablenames
1567 for field in fields:
1568 if isinstance(field, basestring) \
1569 and REGEX_TABLE_DOT_FIELD.match(field):
1570 tn,fn = field.split('.')
1571 field = self.db[tn][fn]
1572 for tablename in tables(field):
1573 if not tablename in tablenames:
1574 tablenames.append(tablename)
1575
1576 if len(tablenames) < 1:
1577 raise SyntaxError('Set: no tables selected')
1578 self._colnames = map(self.expand, fields)
1579 def geoexpand(field):
1580 if isinstance(field.type,str) and field.type.startswith('geometry'):
1581 field = field.st_astext()
1582 return self.expand(field)
1583 sql_f = ', '.join(map(geoexpand, fields))
1584 sql_o = ''
1585 sql_s = ''
1586 left = args_get('left', False)
1587 inner_join = args_get('join', False)
1588 distinct = args_get('distinct', False)
1589 groupby = args_get('groupby', False)
1590 orderby = args_get('orderby', False)
1591 having = args_get('having', False)
1592 limitby = args_get('limitby', False)
1593 orderby_on_limitby = args_get('orderby_on_limitby', True)
1594 for_update = args_get('for_update', False)
1595 if self.can_select_for_update is False and for_update is True:
1596 raise SyntaxError('invalid select attribute: for_update')
1597 if distinct is True:
1598 sql_s += 'DISTINCT'
1599 elif distinct:
1600 sql_s += 'DISTINCT ON (%s)' % distinct
1601 if inner_join:
1602 icommand = self.JOIN()
1603 if not isinstance(inner_join, (tuple, list)):
1604 inner_join = [inner_join]
1605 ijoint = [t._tablename for t in inner_join
1606 if not isinstance(t,Expression)]
1607 ijoinon = [t for t in inner_join if isinstance(t, Expression)]
1608 itables_to_merge={}
1609 [itables_to_merge.update(
1610 dict.fromkeys(tables(t))) for t in ijoinon]
1611 ijoinont = [t.first._tablename for t in ijoinon]
1612 [itables_to_merge.pop(t) for t in ijoinont
1613 if t in itables_to_merge]
1614 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys()
1615 iexcluded = [t for t in tablenames
1616 if not t in iimportant_tablenames]
1617 if left:
1618 join = attributes['left']
1619 command = self.LEFT_JOIN()
1620 if not isinstance(join, (tuple, list)):
1621 join = [join]
1622 joint = [t._tablename for t in join
1623 if not isinstance(t, Expression)]
1624 joinon = [t for t in join if isinstance(t, Expression)]
1625
1626 tables_to_merge={}
1627 [tables_to_merge.update(
1628 dict.fromkeys(tables(t))) for t in joinon]
1629 joinont = [t.first._tablename for t in joinon]
1630 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
1631 tablenames_for_common_filters = [t for t in tablenames
1632 if not t in joinont ]
1633 important_tablenames = joint + joinont + tables_to_merge.keys()
1634 excluded = [t for t in tablenames
1635 if not t in important_tablenames ]
1636 else:
1637 excluded = tablenames
1638
1639 if use_common_filters(query):
1640 query = self.common_filter(query,tablenames_for_common_filters)
1641 sql_w = ' WHERE ' + self.expand(query) if query else ''
1642
1643 if inner_join and not left:
1644 sql_t = ', '.join([self.table_alias(t) for t in iexcluded + \
1645 itables_to_merge.keys()])
1646 for t in ijoinon:
1647 sql_t += ' %s %s' % (icommand, t)
1648 elif not inner_join and left:
1649 sql_t = ', '.join([self.table_alias(t) for t in excluded + \
1650 tables_to_merge.keys()])
1651 if joint:
1652 sql_t += ' %s %s' % (command,
1653 ','.join([self.table_alias(t) for t in joint]))
1654 for t in joinon:
1655 sql_t += ' %s %s' % (command, t)
1656 elif inner_join and left:
1657 all_tables_in_query = set(important_tablenames + \
1658 iimportant_tablenames + \
1659 tablenames)
1660 tables_in_joinon = set(joinont + ijoinont)
1661 tables_not_in_joinon = \
1662 all_tables_in_query.difference(tables_in_joinon)
1663 sql_t = ','.join([self.table_alias(t) for t in tables_not_in_joinon])
1664 for t in ijoinon:
1665 sql_t += ' %s %s' % (icommand, t)
1666 if joint:
1667 sql_t += ' %s %s' % (command,
1668 ','.join([self.table_alias(t) for t in joint]))
1669 for t in joinon:
1670 sql_t += ' %s %s' % (command, t)
1671 else:
1672 sql_t = ', '.join(self.table_alias(t) for t in tablenames)
1673 if groupby:
1674 if isinstance(groupby, (list, tuple)):
1675 groupby = xorify(groupby)
1676 sql_o += ' GROUP BY %s' % self.expand(groupby)
1677 if having:
1678 sql_o += ' HAVING %s' % attributes['having']
1679 if orderby:
1680 if isinstance(orderby, (list, tuple)):
1681 orderby = xorify(orderby)
1682 if str(orderby) == '<random>':
1683 sql_o += ' ORDER BY %s' % self.RANDOM()
1684 else:
1685 sql_o += ' ORDER BY %s' % self.expand(orderby)
1686 if (limitby and not groupby and tablenames and orderby_on_limitby and not orderby):
1687 sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in (hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey or [self.db[t]._id.name])])
1688
1689 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
1690 if for_update and self.can_select_for_update is True:
1691 sql = sql.rstrip(';') + ' FOR UPDATE;'
1692 return sql
1693
1694 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1695 if limitby:
1696 (lmin, lmax) = limitby
1697 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
1698 return 'SELECT %s %s FROM %s%s%s;' % \
1699 (sql_s, sql_f, sql_t, sql_w, sql_o)
1700
1702 return self.cursor.fetchall()
1703
1705 args_get = attributes.get
1706 cache = args_get('cache',None)
1707 if not cache:
1708 self.execute(sql)
1709 rows = self._fetchall()
1710 else:
1711 (cache_model, time_expire) = cache
1712 key = self.uri + '/' + sql + '/rows'
1713 if len(key)>200: key = hashlib_md5(key).hexdigest()
1714 def _select_aux2():
1715 self.execute(sql)
1716 return self._fetchall()
1717 rows = cache_model(key,_select_aux2,time_expire)
1718 if isinstance(rows,tuple):
1719 rows = list(rows)
1720 limitby = args_get('limitby', None) or (0,)
1721 rows = self.rowslice(rows,limitby[0],None)
1722 processor = args_get('processor',self.parse)
1723 cacheable = args_get('cacheable',False)
1724 return processor(rows,fields,self._colnames,cacheable=cacheable)
1725
1726 - def select(self, query, fields, attributes):
1727 """
1728 Always returns a Rows object, possibly empty.
1729 """
1730 sql = self._select(query, fields, attributes)
1731 cache = attributes.get('cache', None)
1732 if cache and attributes.get('cacheable',False):
1733 del attributes['cache']
1734 (cache_model, time_expire) = cache
1735 key = self.uri + '/' + sql
1736 if len(key)>200: key = hashlib_md5(key).hexdigest()
1737 args = (sql,fields,attributes)
1738 return cache_model(
1739 key,
1740 lambda self=self,args=args:self._select_aux(*args),
1741 time_expire)
1742 else:
1743 return self._select_aux(sql,fields,attributes)
1744
1745 - def _count(self, query, distinct=None):
1746 tablenames = self.tables(query)
1747 if query:
1748 if use_common_filters(query):
1749 query = self.common_filter(query, tablenames)
1750 sql_w = ' WHERE ' + self.expand(query)
1751 else:
1752 sql_w = ''
1753 sql_t = ','.join(self.table_alias(t) for t in tablenames)
1754 if distinct:
1755 if isinstance(distinct,(list, tuple)):
1756 distinct = xorify(distinct)
1757 sql_d = self.expand(distinct)
1758 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \
1759 (sql_d, sql_t, sql_w)
1760 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1761
1762 - def count(self, query, distinct=None):
1763 self.execute(self._count(query, distinct))
1764 return self.cursor.fetchone()[0]
1765
1777
1781
1785
1791
1794
1797
1800
1803
1806
1808 return '%s_%s__constraint' % (table,fieldname)
1809
1812
1814 if not self.connection: return None
1815 command = a[0]
1816 if hasattr(self,'filter_sql_command'):
1817 command = self.filter_sql_command(command)
1818 if self.db._debug:
1819 LOGGER.debug('SQL: %s' % command)
1820 self.db._lastsql = command
1821 t0 = time.time()
1822 ret = self.cursor.execute(command, *a[1:], **b)
1823 self.db._timings.append((command,time.time()-t0))
1824 del self.db._timings[:-TIMINGSSIZE]
1825 return ret
1826
1829
1831 field_is_type = fieldtype.startswith
1832 if isinstance(obj, CALLABLETYPES):
1833 obj = obj()
1834 if isinstance(fieldtype, SQLCustomType):
1835 value = fieldtype.encoder(obj)
1836 if fieldtype.type in ('string','text', 'json'):
1837 return self.adapt(value)
1838 return value
1839 if isinstance(obj, (Expression, Field)):
1840 return str(obj)
1841 if field_is_type('list:'):
1842 if not obj:
1843 obj = []
1844 elif not isinstance(obj, (list, tuple)):
1845 obj = [obj]
1846 if field_is_type('list:string'):
1847 obj = map(str,obj)
1848 else:
1849 obj = map(int,[o for o in obj if o != ''])
1850
1851 if isinstance(obj, (list, tuple)) and (not fieldtype == "json"):
1852 obj = bar_encode(obj)
1853 if obj is None:
1854 return 'NULL'
1855 if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']:
1856 return 'NULL'
1857 r = self.represent_exceptions(obj, fieldtype)
1858 if not r is None:
1859 return r
1860 if fieldtype == 'boolean':
1861 if obj and not str(obj)[:1].upper() in '0F':
1862 return self.smart_adapt(self.TRUE)
1863 else:
1864 return self.smart_adapt(self.FALSE)
1865 if fieldtype == 'id' or fieldtype == 'integer':
1866 return str(long(obj))
1867 if field_is_type('decimal'):
1868 return str(obj)
1869 elif field_is_type('reference'):
1870 if fieldtype.find('.')>0:
1871 return repr(obj)
1872 elif isinstance(obj, (Row, Reference)):
1873 return str(obj['id'])
1874 return str(long(obj))
1875 elif fieldtype == 'double':
1876 return repr(float(obj))
1877 if isinstance(obj, unicode):
1878 obj = obj.encode(self.db_codec)
1879 if fieldtype == 'blob':
1880 obj = base64.b64encode(str(obj))
1881 elif fieldtype == 'date':
1882 if isinstance(obj, (datetime.date, datetime.datetime)):
1883 obj = obj.isoformat()[:10]
1884 else:
1885 obj = str(obj)
1886 elif fieldtype == 'datetime':
1887 if isinstance(obj, datetime.datetime):
1888 obj = obj.isoformat(self.T_SEP)[:19]
1889 elif isinstance(obj, datetime.date):
1890 obj = obj.isoformat()[:10]+self.T_SEP+'00:00:00'
1891 else:
1892 obj = str(obj)
1893 elif fieldtype == 'time':
1894 if isinstance(obj, datetime.time):
1895 obj = obj.isoformat()[:10]
1896 else:
1897 obj = str(obj)
1898 elif fieldtype == 'json':
1899 if not self.native_json:
1900 if have_serializers:
1901 obj = serializers.json(obj)
1902 elif simplejson:
1903 obj = simplejson.dumps(obj)
1904 else:
1905 raise RuntimeError("missing simplejson")
1906 if not isinstance(obj,bytes):
1907 obj = bytes(obj)
1908 try:
1909 obj.decode(self.db_codec)
1910 except:
1911 obj = obj.decode('latin1').encode(self.db_codec)
1912 return self.adapt(obj)
1913
1916
1919
1920 - def rowslice(self, rows, minimum=0, maximum=None):
1921 """
1922 By default this function does nothing;
1923 overload when db does not do slicing.
1924 """
1925 return rows
1926
1927 - def parse_value(self, value, field_type, blob_decode=True):
1928 if field_type != 'blob' and isinstance(value, str):
1929 try:
1930 value = value.decode(self.db._db_codec)
1931 except Exception:
1932 pass
1933 if isinstance(value, unicode):
1934 value = value.encode('utf-8')
1935 if isinstance(field_type, SQLCustomType):
1936 value = field_type.decoder(value)
1937 if not isinstance(field_type, str) or value is None:
1938 return value
1939 elif field_type in ('string', 'text', 'password', 'upload', 'dict'):
1940 return value
1941 elif field_type.startswith('geo'):
1942 return value
1943 elif field_type == 'blob' and not blob_decode:
1944 return value
1945 else:
1946 key = REGEX_TYPE.match(field_type).group(0)
1947 return self.parsemap[key](value,field_type)
1948
1950 referee = field_type[10:].strip()
1951 if not '.' in referee:
1952 value = Reference(value)
1953 value._table, value._record = self.db[referee], None
1954 return value
1955
1957 return value == self.TRUE or str(value)[:1].lower() == 't'
1958
1960 if isinstance(value, datetime.datetime):
1961 return value.date()
1962 if not isinstance(value, (datetime.date,datetime.datetime)):
1963 (y, m, d) = map(int, str(value)[:10].strip().split('-'))
1964 value = datetime.date(y, m, d)
1965 return value
1966
1968 if not isinstance(value, datetime.time):
1969 time_items = map(int,str(value)[:8].strip().split(':')[:3])
1970 if len(time_items) == 3:
1971 (h, mi, s) = time_items
1972 else:
1973 (h, mi, s) = time_items + [0]
1974 value = datetime.time(h, mi, s)
1975 return value
1976
1978 if not isinstance(value, datetime.datetime):
1979 value = str(value)
1980 date_part,time_part,timezone = value[:10],value[11:19],value[19:]
1981 if '+' in timezone:
1982 ms,tz = timezone.split('+')
1983 h,m = tz.split(':')
1984 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m))
1985 elif '-' in timezone:
1986 ms,tz = timezone.split('-')
1987 h,m = tz.split(':')
1988 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m))
1989 else:
1990 dt = None
1991 (y, m, d) = map(int,date_part.split('-'))
1992 time_parts = time_part and time_part.split(':')[:3] or (0,0,0)
1993 while len(time_parts)<3: time_parts.append(0)
1994 time_items = map(int,time_parts)
1995 (h, mi, s) = time_items
1996 value = datetime.datetime(y, m, d, h, mi, s)
1997 if dt:
1998 value = value + dt
1999 return value
2000
2002 return base64.b64decode(str(value))
2003
2005 decimals = int(field_type[8:-1].split(',')[-1])
2006 if self.dbengine in ('sqlite', 'spatialite'):
2007 value = ('%.' + str(decimals) + 'f') % value
2008 if not isinstance(value, decimal.Decimal):
2009 value = decimal.Decimal(str(value))
2010 return value
2011
2016
2021
2026
2027 - def parse_id(self, value, field_type):
2029
2032
2035
2037 if not self.native_json:
2038 if not isinstance(value, basestring):
2039 raise RuntimeError('json data not a string')
2040 if isinstance(value, unicode):
2041 value = value.encode('utf-8')
2042 if have_serializers:
2043 value = serializers.loads_json(value)
2044 elif simplejson:
2045 value = simplejson.loads(value)
2046 else:
2047 raise RuntimeError("missing simplejson")
2048 return value
2049
2051 self.parsemap = {
2052 'id':self.parse_id,
2053 'integer':self.parse_integer,
2054 'bigint':self.parse_integer,
2055 'float':self.parse_double,
2056 'double':self.parse_double,
2057 'reference':self.parse_reference,
2058 'boolean':self.parse_boolean,
2059 'date':self.parse_date,
2060 'time':self.parse_time,
2061 'datetime':self.parse_datetime,
2062 'blob':self.parse_blob,
2063 'decimal':self.parse_decimal,
2064 'json':self.parse_json,
2065 'list:integer':self.parse_list_integers,
2066 'list:reference':self.parse_list_references,
2067 'list:string':self.parse_list_strings,
2068 }
2069
2070 - def parse(self, rows, fields, colnames, blob_decode=True,
2071 cacheable = False):
2072 db = self.db
2073 virtualtables = []
2074 new_rows = []
2075 tmps = []
2076 for colname in colnames:
2077 if not REGEX_TABLE_DOT_FIELD.match(colname):
2078 tmps.append(None)
2079 else:
2080 (tablename, _the_sep_, fieldname) = colname.partition('.')
2081 table = db[tablename]
2082 field = table[fieldname]
2083 ft = field.type
2084 tmps.append((tablename,fieldname,table,field,ft))
2085 for (i,row) in enumerate(rows):
2086 new_row = Row()
2087 for (j,colname) in enumerate(colnames):
2088 value = row[j]
2089 tmp = tmps[j]
2090 if tmp:
2091 (tablename,fieldname,table,field,ft) = tmp
2092 if tablename in new_row:
2093 colset = new_row[tablename]
2094 else:
2095 colset = new_row[tablename] = Row()
2096 if tablename not in virtualtables:
2097 virtualtables.append(tablename)
2098 value = self.parse_value(value,ft,blob_decode)
2099 if field.filter_out:
2100 value = field.filter_out(value)
2101 colset[fieldname] = value
2102
2103
2104 if ft=='id' and fieldname!='id' and \
2105 not 'id' in table.fields:
2106 colset['id'] = value
2107
2108 if ft == 'id' and not cacheable:
2109
2110
2111
2112 if isinstance(self, GoogleDatastoreAdapter):
2113 id = value.key().id_or_name()
2114 colset[fieldname] = id
2115 colset.gae_item = value
2116 else:
2117 id = value
2118 colset.update_record = RecordUpdater(colset,table,id)
2119 colset.delete_record = RecordDeleter(table,id)
2120 if table._db._lazy_tables:
2121 colset['__get_lazy_reference__'] = LazyReferenceGetter(table, id)
2122 for rfield in table._referenced_by:
2123 referee_link = db._referee_name and \
2124 db._referee_name % dict(
2125 table=rfield.tablename,field=rfield.name)
2126 if referee_link and not referee_link in colset:
2127 colset[referee_link] = LazySet(rfield,id)
2128 else:
2129 if not '_extra' in new_row:
2130 new_row['_extra'] = Row()
2131 new_row['_extra'][colname] = \
2132 self.parse_value(value,
2133 fields[j].type,blob_decode)
2134 new_column_name = \
2135 REGEX_SELECT_AS_PARSER.search(colname)
2136 if not new_column_name is None:
2137 column_name = new_column_name.groups(0)
2138 setattr(new_row,column_name[0],value)
2139 new_rows.append(new_row)
2140 rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
2141
2142
2143 for tablename in virtualtables:
2144 table = db[tablename]
2145 fields_virtual = [(f,v) for (f,v) in table.iteritems()
2146 if isinstance(v,FieldVirtual)]
2147 fields_lazy = [(f,v) for (f,v) in table.iteritems()
2148 if isinstance(v,FieldMethod)]
2149 if fields_virtual or fields_lazy:
2150 for row in rowsobj.records:
2151 box = row[tablename]
2152 for f,v in fields_virtual:
2153 box[f] = v.f(row)
2154 for f,v in fields_lazy:
2155 box[f] = (v.handler or VirtualCommand)(v.f,row)
2156
2157
2158 for item in table.virtualfields:
2159 try:
2160 rowsobj = rowsobj.setvirtualfields(**{tablename:item})
2161 except (KeyError, AttributeError):
2162
2163 pass
2164 return rowsobj
2165
2167 tenant_fieldname = self.db._request_tenant
2168
2169 for tablename in tablenames:
2170 table = self.db[tablename]
2171
2172
2173 if table._common_filter != None:
2174 query = query & table._common_filter(query)
2175
2176
2177 if tenant_fieldname in table:
2178 default = table[tenant_fieldname].default
2179 if not default is None:
2180 newquery = table[tenant_fieldname] == default
2181 if query is None:
2182 query = newquery
2183 else:
2184 query = query & newquery
2185 return query
2186
2187 - def CASE(self,query,t,f):
2188 def represent(x):
2189 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'}
2190 if x is None: return 'NULL'
2191 elif isinstance(x,Expression): return str(x)
2192 else: return self.represent(x,types.get(type(x),'string'))
2193 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \
2194 (self.expand(query),represent(t),represent(f)))
2195
2201 drivers = ('sqlite2','sqlite3')
2202
2203 can_select_for_update = None
2204
2206 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2207
2208 @staticmethod
2210 table = {
2211 'year': (0, 4),
2212 'month': (5, 7),
2213 'day': (8, 10),
2214 'hour': (11, 13),
2215 'minute': (14, 16),
2216 'second': (17, 19),
2217 }
2218 try:
2219 if lookup != 'epoch':
2220 (i, j) = table[lookup]
2221 return int(s[i:j])
2222 else:
2223 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
2224 except:
2225 return None
2226
2227 @staticmethod
2229 return re.compile(expression).search(item) is not None
2230
2231 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2232 credential_decoder=IDENTITY, driver_args={},
2233 adapter_args={}, do_connect=True, after_connection=None):
2234 self.db = db
2235 self.dbengine = "sqlite"
2236 self.uri = uri
2237 if do_connect: self.find_driver(adapter_args)
2238 self.pool_size = 0
2239 self.folder = folder
2240 self.db_codec = db_codec
2241 self._after_connection = after_connection
2242 self.find_or_make_work_folder()
2243 path_encoding = sys.getfilesystemencoding() \
2244 or locale.getdefaultlocale()[1] or 'utf8'
2245 if uri.startswith('sqlite:memory'):
2246 self.dbpath = ':memory:'
2247 else:
2248 self.dbpath = uri.split('://',1)[1]
2249 if self.dbpath[0] != '/':
2250 if PYTHON_VERSION == 2:
2251 self.dbpath = pjoin(
2252 self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
2253 else:
2254 self.dbpath = pjoin(self.folder, self.dbpath)
2255 if not 'check_same_thread' in driver_args:
2256 driver_args['check_same_thread'] = False
2257 if not 'detect_types' in driver_args and do_connect:
2258 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2259 def connector(dbpath=self.dbpath, driver_args=driver_args):
2260 return self.driver.Connection(dbpath, **driver_args)
2261 self.connector = connector
2262 if do_connect: self.reconnect()
2263
2269
2271 tablename = table._tablename
2272 return ['DELETE FROM %s;' % tablename,
2273 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2274
2277
2278 - def REGEXP(self,first,second):
2279 return '(%s REGEXP %s)' % (self.expand(first),
2280 self.expand(second,'string'))
2281
2282 - def select(self, query, fields, attributes):
2283 """
2284 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
2285 Note that the entire database, rather than one record, is locked
2286 (it will be locked eventually anyway by the following UPDATE).
2287 """
2288 if attributes.get('for_update', False) and not 'cache' in attributes:
2289 self.execute('BEGIN IMMEDIATE TRANSACTION;')
2290 return super(SQLiteAdapter, self).select(query, fields, attributes)
2291
2293 drivers = ('sqlite3','sqlite2')
2294
2295 types = copy.copy(BaseAdapter.types)
2296 types.update(geometry='GEOMETRY')
2297
2298 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2299 credential_decoder=IDENTITY, driver_args={},
2300 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
2301 self.db = db
2302 self.dbengine = "spatialite"
2303 self.uri = uri
2304 if do_connect: self.find_driver(adapter_args)
2305 self.pool_size = 0
2306 self.folder = folder
2307 self.db_codec = db_codec
2308 self._after_connection = after_connection
2309 self.find_or_make_work_folder()
2310 self.srid = srid
2311 path_encoding = sys.getfilesystemencoding() \
2312 or locale.getdefaultlocale()[1] or 'utf8'
2313 if uri.startswith('spatialite:memory'):
2314 self.dbpath = ':memory:'
2315 else:
2316 self.dbpath = uri.split('://',1)[1]
2317 if self.dbpath[0] != '/':
2318 self.dbpath = pjoin(
2319 self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
2320 if not 'check_same_thread' in driver_args:
2321 driver_args['check_same_thread'] = False
2322 if not 'detect_types' in driver_args and do_connect:
2323 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2324 def connector(dbpath=self.dbpath, driver_args=driver_args):
2325 return self.driver.Connection(dbpath, **driver_args)
2326 self.connector = connector
2327 if do_connect: self.reconnect()
2328
2341
2342
2343
2345 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
2346 second['precision'], second['options'])
2347
2348 - def ST_ASTEXT(self, first):
2349 return 'AsText(%s)' %(self.expand(first))
2350
2354
2358
2362
2366
2370
2372 return 'Simplify(%s,%s)' %(self.expand(first),
2373 self.expand(second, 'double'))
2374
2378
2382
2384 field_is_type = fieldtype.startswith
2385 if field_is_type('geo'):
2386 srid = 4326
2387 geotype, parms = fieldtype[:-1].split('(')
2388 parms = parms.split(',')
2389 if len(parms) >= 2:
2390 schema, srid = parms[:2]
2391
2392 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2393
2394
2395
2396
2397 return value
2398 return BaseAdapter.represent(self, obj, fieldtype)
2399
2402 drivers = ('zxJDBC_sqlite',)
2403
2404 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
2405 credential_decoder=IDENTITY, driver_args={},
2406 adapter_args={}, do_connect=True, after_connection=None):
2429 self.connector = connector
2430 if do_connect: self.reconnect()
2431
2436
2439
2442 drivers = ('MySQLdb','pymysql')
2443
2444 commit_on_alter_table = True
2445 support_distributed_transaction = True
2446 types = {
2447 'boolean': 'CHAR(1)',
2448 'string': 'VARCHAR(%(length)s)',
2449 'text': 'LONGTEXT',
2450 'json': 'LONGTEXT',
2451 'password': 'VARCHAR(%(length)s)',
2452 'blob': 'LONGBLOB',
2453 'upload': 'VARCHAR(%(length)s)',
2454 'integer': 'INT',
2455 'bigint': 'BIGINT',
2456 'float': 'FLOAT',
2457 'double': 'DOUBLE',
2458 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2459 'date': 'DATE',
2460 'time': 'TIME',
2461 'datetime': 'DATETIME',
2462 'id': 'INT AUTO_INCREMENT NOT NULL',
2463 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2464 'list:integer': 'LONGTEXT',
2465 'list:string': 'LONGTEXT',
2466 'list:reference': 'LONGTEXT',
2467 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
2468 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2469 }
2470
2471 QUOTE_TEMPLATE = "`%s`"
2472
2475
2478
2480 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
2481 parameters[0], parameters[1])
2482
2483 - def EPOCH(self, first):
2485
2487 return 'CONCAT(%s)' % ','.join(self.expand(x,'string') for x in items)
2488
2489 - def REGEXP(self,first,second):
2490 return '(%s REGEXP %s)' % (self.expand(first),
2491 self.expand(second,'string'))
2492
2493 - def _drop(self,table,mode):
2494
2495 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,
2496 'SET FOREIGN_KEY_CHECKS=1;']
2497
2499 return 'INSERT INTO %s VALUES (DEFAULT);' % table
2500
2503
2507
2510
2513
2514 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
2515
2516 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2517 credential_decoder=IDENTITY, driver_args={},
2518 adapter_args={}, do_connect=True, after_connection=None):
2519 self.db = db
2520 self.dbengine = "mysql"
2521 self.uri = uri
2522 if do_connect: self.find_driver(adapter_args,uri)
2523 self.pool_size = pool_size
2524 self.folder = folder
2525 self.db_codec = db_codec
2526 self._after_connection = after_connection
2527 self.find_or_make_work_folder()
2528 ruri = uri.split('://',1)[1]
2529 m = self.REGEX_URI.match(ruri)
2530 if not m:
2531 raise SyntaxError(
2532 "Invalid URI string in DAL: %s" % self.uri)
2533 user = credential_decoder(m.group('user'))
2534 if not user:
2535 raise SyntaxError('User required')
2536 password = credential_decoder(m.group('password'))
2537 if not password:
2538 password = ''
2539 host = m.group('host')
2540 if not host:
2541 raise SyntaxError('Host name required')
2542 db = m.group('db')
2543 if not db:
2544 raise SyntaxError('Database name required')
2545 port = int(m.group('port') or '3306')
2546 charset = m.group('charset') or 'utf8'
2547 driver_args.update(db=db,
2548 user=credential_decoder(user),
2549 passwd=credential_decoder(password),
2550 host=host,
2551 port=port,
2552 charset=charset)
2553
2554
2555 def connector(driver_args=driver_args):
2556 return self.driver.connect(**driver_args)
2557 self.connector = connector
2558 if do_connect: self.reconnect()
2559
2561 self.execute('SET FOREIGN_KEY_CHECKS=1;')
2562 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2563
2565 self.execute('select last_insert_id();')
2566 return int(self.cursor.fetchone()[0])
2567
2568
2569 -class PostgreSQLAdapter(BaseAdapter):
2570 drivers = ('psycopg2','pg8000')
2571
2572 support_distributed_transaction = True
2573 types = {
2574 'boolean': 'CHAR(1)',
2575 'string': 'VARCHAR(%(length)s)',
2576 'text': 'TEXT',
2577 'json': 'TEXT',
2578 'password': 'VARCHAR(%(length)s)',
2579 'blob': 'BYTEA',
2580 'upload': 'VARCHAR(%(length)s)',
2581 'integer': 'INTEGER',
2582 'bigint': 'BIGINT',
2583 'float': 'FLOAT',
2584 'double': 'FLOAT8',
2585 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2586 'date': 'DATE',
2587 'time': 'TIME',
2588 'datetime': 'TIMESTAMP',
2589 'id': 'SERIAL PRIMARY KEY',
2590 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2591 'list:integer': 'TEXT',
2592 'list:string': 'TEXT',
2593 'list:reference': 'TEXT',
2594 'geometry': 'GEOMETRY',
2595 'geography': 'GEOGRAPHY',
2596 'big-id': 'BIGSERIAL PRIMARY KEY',
2597 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2598 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2599 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2600
2601 }
2602
2603 QUOTE_TEMPLATE = '%s'
2604
2605 - def varquote(self,name):
2606 return varquote_aux(name,'"%s"')
2607
2608 - def adapt(self,obj):
2609 if self.driver_name == 'psycopg2':
2610 return psycopg2_adapt(obj).getquoted()
2611 elif self.driver_name == 'pg8000':
2612 return "'%s'" % str(obj).replace("%","%%").replace("'","''")
2613 else:
2614 return "'%s'" % str(obj).replace("'","''")
2615
2616 - def sequence_name(self,table):
2617 return '%s_id_Seq' % table
2618
2621
2622 - def ADD(self, first, second):
2623 t = first.type
2624 if t in ('text','string','password', 'json', 'upload','blob'):
2625 return '(%s || %s)' % (self.expand(first), self.expand(second, t))
2626 else:
2627 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2628
2631
2632 - def prepare(self,key):
2633 self.execute("PREPARE TRANSACTION '%s';" % key)
2634
2635 - def commit_prepared(self,key):
2636 self.execute("COMMIT PREPARED '%s';" % key)
2637
2638 - def rollback_prepared(self,key):
2639 self.execute("ROLLBACK PREPARED '%s';" % key)
2640
2641 - def create_sequence_and_triggers(self, query, table, **args):
2642
2643
2644
2645
2646 self.execute(query)
2647
2648 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
2649
2650 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2651 credential_decoder=IDENTITY, driver_args={},
2652 adapter_args={}, do_connect=True, srid=4326,
2653 after_connection=None):
2654 self.db = db
2655 self.dbengine = "postgres"
2656 self.uri = uri
2657 if do_connect: self.find_driver(adapter_args,uri)
2658 self.pool_size = pool_size
2659 self.folder = folder
2660 self.db_codec = db_codec
2661 self._after_connection = after_connection
2662 self.srid = srid
2663 self.find_or_make_work_folder()
2664 ruri = uri.split('://',1)[1]
2665 m = self.REGEX_URI.match(ruri)
2666 if not m:
2667 raise SyntaxError("Invalid URI string in DAL")
2668 user = credential_decoder(m.group('user'))
2669 if not user:
2670 raise SyntaxError('User required')
2671 password = credential_decoder(m.group('password'))
2672 if not password:
2673 password = ''
2674 host = m.group('host')
2675 if not host:
2676 raise SyntaxError('Host name required')
2677 db = m.group('db')
2678 if not db:
2679 raise SyntaxError('Database name required')
2680 port = m.group('port') or '5432'
2681 sslmode = m.group('sslmode')
2682 if sslmode:
2683 msg = ("dbname='%s' user='%s' host='%s' "
2684 "port=%s password='%s' sslmode='%s'") \
2685 % (db, user, host, port, password, sslmode)
2686 else:
2687 msg = ("dbname='%s' user='%s' host='%s' "
2688 "port=%s password='%s'") \
2689 % (db, user, host, port, password)
2690
2691 if self.driver:
2692 self.__version__ = "%s %s" % (self.driver.__name__,
2693 self.driver.__version__)
2694 else:
2695 self.__version__ = None
2696 def connector(msg=msg,driver_args=driver_args):
2697 return self.driver.connect(msg,**driver_args)
2698 self.connector = connector
2699 if do_connect: self.reconnect()
2700
2701 - def after_connection(self):
2702 self.connection.set_client_encoding('UTF8')
2703 self.execute("SET standard_conforming_strings=on;")
2704 self.try_json()
2705
2706 - def lastrowid(self,table):
2707 self.execute("select currval('%s')" % table._sequence_name)
2708 return int(self.cursor.fetchone()[0])
2709
2710 - def try_json(self):
2711
2712
2713 if self.driver_name == "pg8000":
2714 supports_json = self.connection.server_version >= "9.2.0"
2715 elif (self.driver_name == "psycopg2") and \
2716 (self.driver.__version__ >= "2.0.12"):
2717 supports_json = self.connection.server_version >= 90200
2718 elif self.driver_name == "zxJDBC":
2719 supports_json = self.connection.dbversion >= "9.2.0"
2720 else: supports_json = None
2721 if supports_json:
2722 self.types["json"] = "JSON"
2723 self.native_json = True
2724 else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
2725
2726 - def LIKE(self,first,second):
2727 args = (self.expand(first), self.expand(second,'string'))
2728 if not first.type in ('string', 'text', 'json'):
2729 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2730 else:
2731 return '(%s LIKE %s)' % args
2732
2733 - def ILIKE(self,first,second):
2734 args = (self.expand(first), self.expand(second,'string'))
2735 if not first.type in ('string', 'text', 'json'):
2736 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2737 else:
2738 return '(%s ILIKE %s)' % args
2739
2740 - def REGEXP(self,first,second):
2741 return '(%s ~ %s)' % (self.expand(first),
2742 self.expand(second,'string'))
2743
2744 - def STARTSWITH(self,first,second):
2745 return '(%s ILIKE %s)' % (self.expand(first),
2746 self.expand(second+'%','string'))
2747
2748 - def ENDSWITH(self,first,second):
2749 return '(%s ILIKE %s)' % (self.expand(first),
2750 self.expand('%'+second,'string'))
2751
2752
2753
2754 - def ST_ASGEOJSON(self, first, second):
2755 """
2756 http://postgis.org/docs/ST_AsGeoJSON.html
2757 """
2758 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
2759 self.expand(first), second['precision'], second['options'])
2760
2761 - def ST_ASTEXT(self, first):
2762 """
2763 http://postgis.org/docs/ST_AsText.html
2764 """
2765 return 'ST_AsText(%s)' %(self.expand(first))
2766
2767 - def ST_X(self, first):
2768 """
2769 http://postgis.org/docs/ST_X.html
2770 """
2771 return 'ST_X(%s)' %(self.expand(first))
2772
2773 - def ST_Y(self, first):
2774 """
2775 http://postgis.org/docs/ST_Y.html
2776 """
2777 return 'ST_Y(%s)' %(self.expand(first))
2778
2779 - def ST_CONTAINS(self, first, second):
2780 """
2781 http://postgis.org/docs/ST_Contains.html
2782 """
2783 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2784
2785 - def ST_DISTANCE(self, first, second):
2786 """
2787 http://postgis.org/docs/ST_Distance.html
2788 """
2789 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2790
2791 - def ST_EQUALS(self, first, second):
2792 """
2793 http://postgis.org/docs/ST_Equals.html
2794 """
2795 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2796
2797 - def ST_INTERSECTS(self, first, second):
2798 """
2799 http://postgis.org/docs/ST_Intersects.html
2800 """
2801 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2802
2803 - def ST_OVERLAPS(self, first, second):
2804 """
2805 http://postgis.org/docs/ST_Overlaps.html
2806 """
2807 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2808
2809 - def ST_SIMPLIFY(self, first, second):
2810 """
2811 http://postgis.org/docs/ST_Simplify.html
2812 """
2813 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2814
2815 - def ST_TOUCHES(self, first, second):
2816 """
2817 http://postgis.org/docs/ST_Touches.html
2818 """
2819 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2820
2821 - def ST_WITHIN(self, first, second):
2822 """
2823 http://postgis.org/docs/ST_Within.html
2824 """
2825 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2826
2827 - def represent(self, obj, fieldtype):
2828 field_is_type = fieldtype.startswith
2829 if field_is_type('geo'):
2830 srid = 4326
2831 geotype, parms = fieldtype[:-1].split('(')
2832 parms = parms.split(',')
2833 if len(parms) >= 2:
2834 schema, srid = parms[:2]
2835 if field_is_type('geometry'):
2836 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2837 elif field_is_type('geography'):
2838 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
2839
2840
2841 return value
2842 return BaseAdapter.represent(self, obj, fieldtype)
2843
2844 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2845 drivers = ('psycopg2','pg8000')
2846
2847 types = {
2848 'boolean': 'CHAR(1)',
2849 'string': 'VARCHAR(%(length)s)',
2850 'text': 'TEXT',
2851 'json': 'TEXT',
2852 'password': 'VARCHAR(%(length)s)',
2853 'blob': 'BYTEA',
2854 'upload': 'VARCHAR(%(length)s)',
2855 'integer': 'INTEGER',
2856 'bigint': 'BIGINT',
2857 'float': 'FLOAT',
2858 'double': 'FLOAT8',
2859 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2860 'date': 'DATE',
2861 'time': 'TIME',
2862 'datetime': 'TIMESTAMP',
2863 'id': 'SERIAL PRIMARY KEY',
2864 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2865 'list:integer': 'BIGINT[]',
2866 'list:string': 'TEXT[]',
2867 'list:reference': 'BIGINT[]',
2868 'geometry': 'GEOMETRY',
2869 'geography': 'GEOGRAPHY',
2870 'big-id': 'BIGSERIAL PRIMARY KEY',
2871 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2872 }
2873
2874 - def parse_list_integers(self, value, field_type):
2876
2877 - def parse_list_references(self, value, field_type):
2878 return [self.parse_reference(r, field_type[5:]) for r in value]
2879
2880 - def parse_list_strings(self, value, field_type):
2882
2883 - def represent(self, obj, fieldtype):
2884 field_is_type = fieldtype.startswith
2885 if field_is_type('list:'):
2886 if not obj:
2887 obj = []
2888 elif not isinstance(obj, (list, tuple)):
2889 obj = [obj]
2890 if field_is_type('list:string'):
2891 obj = map(str,obj)
2892 else:
2893 obj = map(int,obj)
2894 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
2895 return BaseAdapter.represent(self, obj, fieldtype)
2896
2897
2898 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
2899 drivers = ('zxJDBC',)
2900
2901 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
2902
2903 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2904 credential_decoder=IDENTITY, driver_args={},
2905 adapter_args={}, do_connect=True, after_connection=None ):
2906 self.db = db
2907 self.dbengine = "postgres"
2908 self.uri = uri
2909 if do_connect: self.find_driver(adapter_args,uri)
2910 self.pool_size = pool_size
2911 self.folder = folder
2912 self.db_codec = db_codec
2913 self._after_connection = after_connection
2914 self.find_or_make_work_folder()
2915 ruri = uri.split('://',1)[1]
2916 m = self.REGEX_URI.match(ruri)
2917 if not m:
2918 raise SyntaxError("Invalid URI string in DAL")
2919 user = credential_decoder(m.group('user'))
2920 if not user:
2921 raise SyntaxError('User required')
2922 password = credential_decoder(m.group('password'))
2923 if not password:
2924 password = ''
2925 host = m.group('host')
2926 if not host:
2927 raise SyntaxError('Host name required')
2928 db = m.group('db')
2929 if not db:
2930 raise SyntaxError('Database name required')
2931 port = m.group('port') or '5432'
2932 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
2933 def connector(msg=msg,driver_args=driver_args):
2934 return self.driver.connect(*msg,**driver_args)
2935 self.connector = connector
2936 if do_connect: self.reconnect()
2937
2938 - def after_connection(self):
2939 self.connection.set_client_encoding('UTF8')
2940 self.execute('BEGIN;')
2941 self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
2942 self.try_json()
2943
2946 drivers = ('cx_Oracle',)
2947
2948 commit_on_alter_table = False
2949 types = {
2950 'boolean': 'CHAR(1)',
2951 'string': 'VARCHAR2(%(length)s)',
2952 'text': 'CLOB',
2953 'json': 'CLOB',
2954 'password': 'VARCHAR2(%(length)s)',
2955 'blob': 'CLOB',
2956 'upload': 'VARCHAR2(%(length)s)',
2957 'integer': 'INT',
2958 'bigint': 'NUMBER',
2959 'float': 'FLOAT',
2960 'double': 'BINARY_DOUBLE',
2961 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2962 'date': 'DATE',
2963 'time': 'CHAR(8)',
2964 'datetime': 'DATE',
2965 'id': 'NUMBER PRIMARY KEY',
2966 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2967 'list:integer': 'CLOB',
2968 'list:string': 'CLOB',
2969 'list:reference': 'CLOB',
2970 'big-id': 'NUMBER PRIMARY KEY',
2971 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2972 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2973 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2974 }
2975
2977 return '%s_sequence' % tablename
2978
2980 return '%s_trigger' % tablename
2981
2983 return 'LEFT OUTER JOIN'
2984
2986 return 'dbms_random.value'
2987
2988 - def NOT_NULL(self,default,field_type):
2989 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
2990
2991 - def _drop(self,table,mode):
2994
2995 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
2996 if limitby:
2997 (lmin, lmax) = limitby
2998 if len(sql_w) > 1:
2999 sql_w_row = sql_w + ' AND w_row > %i' % lmin
3000 else:
3001 sql_w_row = 'WHERE w_row > %i' % lmin
3002 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
3003 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3004
3010
3012 if fieldtype == 'blob':
3013 obj = base64.b64encode(str(obj))
3014 return ":CLOB('%s')" % obj
3015 elif fieldtype == 'date':
3016 if isinstance(obj, (datetime.date, datetime.datetime)):
3017 obj = obj.isoformat()[:10]
3018 else:
3019 obj = str(obj)
3020 return "to_date('%s','yyyy-mm-dd')" % obj
3021 elif fieldtype == 'datetime':
3022 if isinstance(obj, datetime.datetime):
3023 obj = obj.isoformat()[:19].replace('T',' ')
3024 elif isinstance(obj, datetime.date):
3025 obj = obj.isoformat()[:10]+' 00:00:00'
3026 else:
3027 obj = str(obj)
3028 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
3029 return None
3030
3031 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3032 credential_decoder=IDENTITY, driver_args={},
3033 adapter_args={}, do_connect=True, after_connection=None):
3034 self.db = db
3035 self.dbengine = "oracle"
3036 self.uri = uri
3037 if do_connect: self.find_driver(adapter_args,uri)
3038 self.pool_size = pool_size
3039 self.folder = folder
3040 self.db_codec = db_codec
3041 self._after_connection = after_connection
3042 self.find_or_make_work_folder()
3043 ruri = uri.split('://',1)[1]
3044 if not 'threaded' in driver_args:
3045 driver_args['threaded']=True
3046 def connector(uri=ruri,driver_args=driver_args):
3047 return self.driver.connect(uri,**driver_args)
3048 self.connector = connector
3049 if do_connect: self.reconnect()
3050
3052 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3053 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3054
3055 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
3056
3057 - def execute(self, command, args=None):
3058 args = args or []
3059 i = 1
3060 while True:
3061 m = self.oracle_fix.match(command)
3062 if not m:
3063 break
3064 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
3065 args.append(m.group('clob')[6:-2].replace("''", "'"))
3066 i += 1
3067 if command[-1:]==';':
3068 command = command[:-1]
3069 return self.log_execute(command, args)
3070
3072 tablename = table._tablename
3073 id_name = table._id.name
3074 sequence_name = table._sequence_name
3075 trigger_name = table._trigger_name
3076 self.execute(query)
3077 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
3078 self.execute("""
3079 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
3080 DECLARE
3081 curr_val NUMBER;
3082 diff_val NUMBER;
3083 PRAGMA autonomous_transaction;
3084 BEGIN
3085 IF :NEW.%(id)s IS NOT NULL THEN
3086 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3087 diff_val := :NEW.%(id)s - curr_val - 1;
3088 IF diff_val != 0 THEN
3089 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
3090 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3091 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
3092 END IF;
3093 END IF;
3094 SELECT %(sequence_name)s.nextval INTO :NEW.%(id)s FROM DUAL;
3095 END;
3096 """ % dict(trigger_name=trigger_name, tablename=tablename,
3097 sequence_name=sequence_name,id=id_name))
3098
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3114 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description):
3115 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
3116 for c in r]) for r in self.cursor]
3117 else:
3118 return self.cursor.fetchall()
3119
3121 drivers = ('pyodbc',)
3122 T_SEP = 'T'
3123
3124 QUOTE_TEMPLATE = "[%s]"
3125
3126 types = {
3127 'boolean': 'BIT',
3128 'string': 'VARCHAR(%(length)s)',
3129 'text': 'TEXT',
3130 'json': 'TEXT',
3131 'password': 'VARCHAR(%(length)s)',
3132 'blob': 'IMAGE',
3133 'upload': 'VARCHAR(%(length)s)',
3134 'integer': 'INT',
3135 'bigint': 'BIGINT',
3136 'float': 'FLOAT',
3137 'double': 'FLOAT',
3138 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3139 'date': 'DATETIME',
3140 'time': 'CHAR(8)',
3141 'datetime': 'DATETIME',
3142 'id': 'INT IDENTITY PRIMARY KEY',
3143 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3144 'list:integer': 'TEXT',
3145 'list:string': 'TEXT',
3146 'list:reference': 'TEXT',
3147 'geometry': 'geometry',
3148 'geography': 'geography',
3149 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3150 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3151 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3152 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3153 }
3154
3156 return '; ALTER TABLE %s ADD ' % tablename
3157
3160
3162 return "DATEPART(%s,%s)" % (what, self.expand(field))
3163
3165 return 'LEFT OUTER JOIN'
3166
3169
3172
3174 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
3175
3177 return 'PRIMARY KEY CLUSTERED (%s)' % key
3178
3180 if what == 'LENGTH':
3181 what = 'LEN'
3182 return "%s(%s)" % (what, self.expand(first))
3183
3184
3185 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3186 if limitby:
3187 (lmin, lmax) = limitby
3188 sql_s += ' TOP %i' % lmax
3189 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3190
3191 TRUE = 1
3192 FALSE = 0
3193
3194 REGEX_DSN = re.compile('^(?P<dsn>.+)$')
3195 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
3196 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
3197
3198 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3199 credential_decoder=IDENTITY, driver_args={},
3200 adapter_args={}, do_connect=True, srid=4326,
3201 after_connection=None):
3202 self.db = db
3203 self.dbengine = "mssql"
3204 self.uri = uri
3205 if do_connect: self.find_driver(adapter_args,uri)
3206 self.pool_size = pool_size
3207 self.folder = folder
3208 self.db_codec = db_codec
3209 self._after_connection = after_connection
3210 self.srid = srid
3211 self.find_or_make_work_folder()
3212
3213 ruri = uri.split('://',1)[1]
3214 if '@' not in ruri:
3215 try:
3216 m = self.REGEX_DSN.match(ruri)
3217 if not m:
3218 raise SyntaxError(
3219 'Parsing uri string(%s) has no result' % self.uri)
3220 dsn = m.group('dsn')
3221 if not dsn:
3222 raise SyntaxError('DSN required')
3223 except SyntaxError:
3224 e = sys.exc_info()[1]
3225 LOGGER.error('NdGpatch error')
3226 raise e
3227
3228 cnxn = dsn
3229 else:
3230 m = self.REGEX_URI.match(ruri)
3231 if not m:
3232 raise SyntaxError(
3233 "Invalid URI string in DAL: %s" % self.uri)
3234 user = credential_decoder(m.group('user'))
3235 if not user:
3236 raise SyntaxError('User required')
3237 password = credential_decoder(m.group('password'))
3238 if not password:
3239 password = ''
3240 host = m.group('host')
3241 if not host:
3242 raise SyntaxError('Host name required')
3243 db = m.group('db')
3244 if not db:
3245 raise SyntaxError('Database name required')
3246 port = m.group('port') or '1433'
3247
3248
3249
3250 argsdict = { 'DRIVER':'{SQL Server}' }
3251 urlargs = m.group('urlargs') or ''
3252 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
3253 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
3254 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
3255 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
3256 % (host, port, db, user, password, urlargs)
3257 def connector(cnxn=cnxn,driver_args=driver_args):
3258 return self.driver.connect(cnxn,**driver_args)
3259 self.connector = connector
3260 if do_connect: self.reconnect()
3261
3263
3264 self.execute('SELECT SCOPE_IDENTITY();')
3265 return long(self.cursor.fetchone()[0])
3266
3267 - def rowslice(self,rows,minimum=0,maximum=None):
3268 if maximum is None:
3269 return rows[minimum:]
3270 return rows[minimum:maximum]
3271
3272 - def EPOCH(self, first):
3273 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3274
3277
3278
3279
3280
3281
3282 - def ST_ASTEXT(self, first):
3283 return '%s.STAsText()' %(self.expand(first))
3284
3287
3290
3293
3296
3299
3300
3301
3304
3307
3309 field_is_type = fieldtype.startswith
3310 if field_is_type('geometry'):
3311 srid = 0
3312 geotype, parms = fieldtype[:-1].split('(')
3313 if parms:
3314 srid = parms
3315 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3316 elif fieldtype == 'geography':
3317 srid = 4326
3318 geotype, parms = fieldtype[:-1].split('(')
3319 if parms:
3320 srid = parms
3321 return "geography::STGeomFromText('%s',%s)" %(obj, srid)
3322
3323
3324 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3325 return BaseAdapter.represent(self, obj, fieldtype)
3326
3329 """ experimental support for pagination in MSSQL"""
3330 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3331 if limitby:
3332 (lmin, lmax) = limitby
3333 if lmin == 0:
3334 sql_s += ' TOP %i' % lmax
3335 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3336 lmin += 1
3337 sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
3338 sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
3339 sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
3340 sql_f_inner = [f for f in sql_f.split(',')]
3341 sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
3342 sql_f_iproxy = ', '.join(sql_f_iproxy)
3343 sql_f_oproxy = ', '.join(sql_f_outer)
3344 return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
3345 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3346 - def rowslice(self,rows,minimum=0,maximum=None):
3348
3351 drivers = ('pyodbc',)
3352
3353 types = {
3354 'boolean': 'CHAR(1)',
3355 'string': 'NVARCHAR(%(length)s)',
3356 'text': 'NTEXT',
3357 'json': 'NTEXT',
3358 'password': 'NVARCHAR(%(length)s)',
3359 'blob': 'IMAGE',
3360 'upload': 'NVARCHAR(%(length)s)',
3361 'integer': 'INT',
3362 'bigint': 'BIGINT',
3363 'float': 'FLOAT',
3364 'double': 'FLOAT',
3365 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3366 'date': 'DATETIME',
3367 'time': 'CHAR(8)',
3368 'datetime': 'DATETIME',
3369 'id': 'INT IDENTITY PRIMARY KEY',
3370 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3371 'list:integer': 'NTEXT',
3372 'list:string': 'NTEXT',
3373 'list:reference': 'NTEXT',
3374 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3375 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3376 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3377 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3378 }
3379
3381 value = BaseAdapter.represent(self, obj, fieldtype)
3382 if fieldtype in ('string','text', 'json') and value[:1]=="'":
3383 value = 'N'+value
3384 return value
3385
3388
3390 drivers = ('pyodbc',)
3391 T_SEP = ' '
3392
3393 types = {
3394 'boolean': 'BOOLEAN',
3395 'string': 'VARCHAR(%(length)s)',
3396 'text': 'BYTEA',
3397 'json': 'VARCHAR(%(length)s)',
3398 'password': 'VARCHAR(%(length)s)',
3399 'blob': 'BYTEA',
3400 'upload': 'VARCHAR(%(length)s)',
3401 'integer': 'INT',
3402 'bigint': 'BIGINT',
3403 'float': 'FLOAT',
3404 'double': 'DOUBLE PRECISION',
3405 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3406 'date': 'DATE',
3407 'time': 'TIME',
3408 'datetime': 'DATETIME',
3409 'id': 'IDENTITY',
3410 'reference': 'INT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3411 'list:integer': 'BYTEA',
3412 'list:string': 'BYTEA',
3413 'list:reference': 'BYTEA',
3414 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3415 }
3416
3417
3419 return "DATE_PART('%s', TIMESTAMP %s)" % (what, self.expand(first))
3420
3422 tablename = table._tablename
3423 return ['TRUNCATE %s %s;' % (tablename, mode or '')]
3424
3425 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3426 if limitby:
3427 (lmin, lmax) = limitby
3428 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
3429 return 'SELECT %s %s FROM %s%s%s;' % \
3430 (sql_s, sql_f, sql_t, sql_w, sql_o)
3431
3433 self.execute('SELECT LAST_INSERT_ID();')
3434 return long(self.cursor.fetchone()[0])
3435
3438
3440 drivers = ('Sybase',)
3441
3442 types = {
3443 'boolean': 'BIT',
3444 'string': 'CHAR VARYING(%(length)s)',
3445 'text': 'TEXT',
3446 'json': 'TEXT',
3447 'password': 'CHAR VARYING(%(length)s)',
3448 'blob': 'IMAGE',
3449 'upload': 'CHAR VARYING(%(length)s)',
3450 'integer': 'INT',
3451 'bigint': 'BIGINT',
3452 'float': 'FLOAT',
3453 'double': 'FLOAT',
3454 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3455 'date': 'DATETIME',
3456 'time': 'CHAR(8)',
3457 'datetime': 'DATETIME',
3458 'id': 'INT IDENTITY PRIMARY KEY',
3459 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3460 'list:integer': 'TEXT',
3461 'list:string': 'TEXT',
3462 'list:reference': 'TEXT',
3463 'geometry': 'geometry',
3464 'geography': 'geography',
3465 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3466 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3467 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3468 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3469 }
3470
3471
3472 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3473 credential_decoder=IDENTITY, driver_args={},
3474 adapter_args={}, do_connect=True, srid=4326,
3475 after_connection=None):
3476 self.db = db
3477 self.dbengine = "sybase"
3478 self.uri = uri
3479 if do_connect: self.find_driver(adapter_args,uri)
3480 self.pool_size = pool_size
3481 self.folder = folder
3482 self.db_codec = db_codec
3483 self._after_connection = after_connection
3484 self.srid = srid
3485 self.find_or_make_work_folder()
3486
3487 ruri = uri.split('://',1)[1]
3488 if '@' not in ruri:
3489 try:
3490 m = self.REGEX_DSN.match(ruri)
3491 if not m:
3492 raise SyntaxError(
3493 'Parsing uri string(%s) has no result' % self.uri)
3494 dsn = m.group('dsn')
3495 if not dsn:
3496 raise SyntaxError('DSN required')
3497 except SyntaxError:
3498 e = sys.exc_info()[1]
3499 LOGGER.error('NdGpatch error')
3500 raise e
3501 else:
3502 m = self.REGEX_URI.match(uri)
3503 if not m:
3504 raise SyntaxError(
3505 "Invalid URI string in DAL: %s" % self.uri)
3506 user = credential_decoder(m.group('user'))
3507 if not user:
3508 raise SyntaxError('User required')
3509 password = credential_decoder(m.group('password'))
3510 if not password:
3511 password = ''
3512 host = m.group('host')
3513 if not host:
3514 raise SyntaxError('Host name required')
3515 db = m.group('db')
3516 if not db:
3517 raise SyntaxError('Database name required')
3518 port = m.group('port') or '1433'
3519
3520 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
3521
3522 driver_args.update(user = credential_decoder(user),
3523 password = credential_decoder(password))
3524
3525 def connector(dsn=dsn,driver_args=driver_args):
3526 return self.driver.connect(dsn,**driver_args)
3527 self.connector = connector
3528 if do_connect: self.reconnect()
3529
3532 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3533
3534 commit_on_alter_table = False
3535 support_distributed_transaction = True
3536 types = {
3537 'boolean': 'CHAR(1)',
3538 'string': 'VARCHAR(%(length)s)',
3539 'text': 'BLOB SUB_TYPE 1',
3540 'json': 'BLOB SUB_TYPE 1',
3541 'password': 'VARCHAR(%(length)s)',
3542 'blob': 'BLOB SUB_TYPE 0',
3543 'upload': 'VARCHAR(%(length)s)',
3544 'integer': 'INTEGER',
3545 'bigint': 'BIGINT',
3546 'float': 'FLOAT',
3547 'double': 'DOUBLE PRECISION',
3548 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3549 'date': 'DATE',
3550 'time': 'TIME',
3551 'datetime': 'TIMESTAMP',
3552 'id': 'INTEGER PRIMARY KEY',
3553 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3554 'list:integer': 'BLOB SUB_TYPE 1',
3555 'list:string': 'BLOB SUB_TYPE 1',
3556 'list:reference': 'BLOB SUB_TYPE 1',
3557 'big-id': 'BIGINT PRIMARY KEY',
3558 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3559 }
3560
3562 return 'genid_%s' % tablename
3563
3565 return 'trg_id_%s' % tablename
3566
3569
3570 - def EPOCH(self, first):
3571 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3572
3573 - def NOT_NULL(self,default,field_type):
3574 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3575
3577 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3578
3581
3582 - def CONTAINS(self,first,second,case_sensitive=False):
3588
3589 - def _drop(self,table,mode):
3592
3593 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3594 if limitby:
3595 (lmin, lmax) = limitby
3596 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
3597 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3598
3600 return ['DELETE FROM %s;' % table._tablename,
3601 'SET GENERATOR %s TO 0;' % table._sequence_name]
3602
3603 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
3604
3605 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3606 credential_decoder=IDENTITY, driver_args={},
3607 adapter_args={}, do_connect=True, after_connection=None):
3608 self.db = db
3609 self.dbengine = "firebird"
3610 self.uri = uri
3611 if do_connect: self.find_driver(adapter_args,uri)
3612 self.pool_size = pool_size
3613 self.folder = folder
3614 self.db_codec = db_codec
3615 self._after_connection = after_connection
3616 self.find_or_make_work_folder()
3617 ruri = uri.split('://',1)[1]
3618 m = self.REGEX_URI.match(ruri)
3619 if not m:
3620 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
3621 user = credential_decoder(m.group('user'))
3622 if not user:
3623 raise SyntaxError('User required')
3624 password = credential_decoder(m.group('password'))
3625 if not password:
3626 password = ''
3627 host = m.group('host')
3628 if not host:
3629 raise SyntaxError('Host name required')
3630 port = int(m.group('port') or 3050)
3631 db = m.group('db')
3632 if not db:
3633 raise SyntaxError('Database name required')
3634 charset = m.group('charset') or 'UTF8'
3635 driver_args.update(dsn='%s/%s:%s' % (host,port,db),
3636 user = credential_decoder(user),
3637 password = credential_decoder(password),
3638 charset = charset)
3639
3640 def connector(driver_args=driver_args):
3641 return self.driver.connect(**driver_args)
3642 self.connector = connector
3643 if do_connect: self.reconnect()
3644
3653
3658
3661 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3662
3663 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
3664
3665 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3666 credential_decoder=IDENTITY, driver_args={},
3667 adapter_args={}, do_connect=True, after_connection=None):
3668 self.db = db
3669 self.dbengine = "firebird"
3670 self.uri = uri
3671 if do_connect: self.find_driver(adapter_args,uri)
3672 self.pool_size = pool_size
3673 self.folder = folder
3674 self.db_codec = db_codec
3675 self._after_connection = after_connection
3676 self.find_or_make_work_folder()
3677 ruri = uri.split('://',1)[1]
3678 m = self.REGEX_URI.match(ruri)
3679 if not m:
3680 raise SyntaxError(
3681 "Invalid URI string in DAL: %s" % self.uri)
3682 user = credential_decoder(m.group('user'))
3683 if not user:
3684 raise SyntaxError('User required')
3685 password = credential_decoder(m.group('password'))
3686 if not password:
3687 password = ''
3688 pathdb = m.group('path')
3689 if not pathdb:
3690 raise SyntaxError('Path required')
3691 charset = m.group('charset')
3692 if not charset:
3693 charset = 'UTF8'
3694 host = ''
3695 driver_args.update(host=host,
3696 database=pathdb,
3697 user=credential_decoder(user),
3698 password=credential_decoder(password),
3699 charset=charset)
3700
3701 def connector(driver_args=driver_args):
3702 return self.driver.connect(**driver_args)
3703 self.connector = connector
3704 if do_connect: self.reconnect()
3705
3811
3816
3819
3831
3833 drivers = ('pyodbc',)
3834
3835 types = {
3836 'boolean': 'CHAR(1)',
3837 'string': 'VARCHAR(%(length)s)',
3838 'text': 'CLOB',
3839 'json': 'CLOB',
3840 'password': 'VARCHAR(%(length)s)',
3841 'blob': 'BLOB',
3842 'upload': 'VARCHAR(%(length)s)',
3843 'integer': 'INT',
3844 'bigint': 'BIGINT',
3845 'float': 'REAL',
3846 'double': 'DOUBLE',
3847 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3848 'date': 'DATE',
3849 'time': 'TIME',
3850 'datetime': 'TIMESTAMP',
3851 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3852 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3853 'list:integer': 'CLOB',
3854 'list:string': 'CLOB',
3855 'list:reference': 'CLOB',
3856 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3857 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3858 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3859 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3860 }
3861
3863 return 'LEFT OUTER JOIN'
3864
3867
3868 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3869 if limitby:
3870 (lmin, lmax) = limitby
3871 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
3872 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3873
3875 if fieldtype == 'blob':
3876 obj = base64.b64encode(str(obj))
3877 return "BLOB('%s')" % obj
3878 elif fieldtype == 'datetime':
3879 if isinstance(obj, datetime.datetime):
3880 obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
3881 elif isinstance(obj, datetime.date):
3882 obj = obj.isoformat()[:10]+'-00.00.00'
3883 return "'%s'" % obj
3884 return None
3885
3886 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3887 credential_decoder=IDENTITY, driver_args={},
3888 adapter_args={}, do_connect=True, after_connection=None):
3901 self.connector = connector
3902 if do_connect: self.reconnect()
3903
3905 if command[-1:]==';':
3906 command = command[:-1]
3907 return self.log_execute(command)
3908
3910 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
3911 return long(self.cursor.fetchone()[0])
3912
3913 - def rowslice(self,rows,minimum=0,maximum=None):
3914 if maximum is None:
3915 return rows[minimum:]
3916 return rows[minimum:maximum]
3917
3920 drivers = ('pyodbc',)
3921
3922 types = {
3923 'boolean': 'CHAR(1)',
3924 'string': 'VARCHAR(%(length)s)',
3925 'text': 'CLOB',
3926 'json': 'CLOB',
3927 'password': 'VARCHAR(%(length)s)',
3928 'blob': 'BLOB',
3929 'upload': 'VARCHAR(%(length)s)',
3930 'integer': 'INT',
3931 'bigint': 'BIGINT',
3932 'float': 'REAL',
3933 'double': 'DOUBLE',
3934 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3935 'date': 'DATE',
3936 'time': 'TIME',
3937 'datetime': 'TIMESTAMP',
3938
3939
3940 'id': 'INT GENERATED ALWAYS AS IDENTITY',
3941 'reference': 'INT',
3942 'list:integer': 'CLOB',
3943 'list:string': 'CLOB',
3944 'list:reference': 'CLOB',
3945 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',
3946 'big-reference': 'BIGINT',
3947 'reference FK': ' REFERENCES %(foreign_key)s',
3948 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
3949 }
3950
3951 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3952 credential_decoder=IDENTITY, driver_args={},
3953 adapter_args={}, do_connect=True, after_connection=None):
3966 self.connector = connector
3967 if do_connect: self.reconnect()
3968
3970 return 'LEFT OUTER JOIN'
3971
3972
3973 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3974 if limitby:
3975 (lmin, lmax) = limitby
3976 sql_s += ' TOP %i' % lmax
3977 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3978
3980 tablename = table._tablename
3981 return ['DELETE FROM %s ALL;' % (tablename)]
3982
3983 INGRES_SEQNAME='ii***lineitemsequence'
3988 drivers = ('pyodbc',)
3989
3990 types = {
3991 'boolean': 'CHAR(1)',
3992 'string': 'VARCHAR(%(length)s)',
3993 'text': 'CLOB',
3994 'json': 'CLOB',
3995 'password': 'VARCHAR(%(length)s)',
3996 'blob': 'BLOB',
3997 'upload': 'VARCHAR(%(length)s)',
3998 'integer': 'INTEGER4',
3999 'bigint': 'BIGINT',
4000 'float': 'FLOAT',
4001 'double': 'FLOAT8',
4002 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4003 'date': 'ANSIDATE',
4004 'time': 'TIME WITHOUT TIME ZONE',
4005 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
4006 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
4007 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4008 'list:integer': 'CLOB',
4009 'list:string': 'CLOB',
4010 'list:reference': 'CLOB',
4011 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
4012 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4013 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4014 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4015 }
4016
4018 return 'LEFT OUTER JOIN'
4019
4022
4023 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4024 if limitby:
4025 (lmin, lmax) = limitby
4026 fetch_amt = lmax - lmin
4027 if fetch_amt:
4028 sql_s += ' FIRST %d ' % (fetch_amt, )
4029 if lmin:
4030
4031 sql_o += ' OFFSET %d' % (lmin, )
4032 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4033
4034 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4035 credential_decoder=IDENTITY, driver_args={},
4036 adapter_args={}, do_connect=True, after_connection=None):
4037 self.db = db
4038 self.dbengine = "ingres"
4039 self._driver = pyodbc
4040 self.uri = uri
4041 if do_connect: self.find_driver(adapter_args,uri)
4042 self.pool_size = pool_size
4043 self.folder = folder
4044 self.db_codec = db_codec
4045 self._after_connection = after_connection
4046 self.find_or_make_work_folder()
4047 connstr = uri.split(':', 1)[1]
4048
4049 connstr = connstr.lstrip()
4050 while connstr.startswith('/'):
4051 connstr = connstr[1:]
4052 if '=' in connstr:
4053
4054 ruri = connstr
4055 else:
4056
4057 database_name = connstr
4058 default_driver_name = 'Ingres'
4059 vnode = '(local)'
4060 servertype = 'ingres'
4061 ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name)
4062 def connector(cnxn=ruri,driver_args=driver_args):
4063 return self.driver.connect(cnxn,**driver_args)
4064
4065 self.connector = connector
4066
4067
4068 if do_connect: self.reconnect()
4069
4071
4072
4073
4074 if hasattr(table,'_primarykey'):
4075 modify_tbl_sql = 'modify %s to btree unique on %s' % \
4076 (table._tablename,
4077 ', '.join(["'%s'" % x for x in table.primarykey]))
4078 self.execute(modify_tbl_sql)
4079 else:
4080 tmp_seqname='%s_iisq' % table._tablename
4081 query=query.replace(INGRES_SEQNAME, tmp_seqname)
4082 self.execute('create sequence %s' % tmp_seqname)
4083 self.execute(query)
4084 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
4085
4086
4088 tmp_seqname='%s_iisq' % table
4089 self.execute('select current value for %s' % tmp_seqname)
4090 return long(self.cursor.fetchone()[0])
4091
4094
4095 drivers = ('pyodbc',)
4096
4097 types = {
4098 'boolean': 'CHAR(1)',
4099 'string': 'NVARCHAR(%(length)s)',
4100 'text': 'NCLOB',
4101 'json': 'NCLOB',
4102 'password': 'NVARCHAR(%(length)s)',
4103 'blob': 'BLOB',
4104 'upload': 'VARCHAR(%(length)s)',
4105 'integer': 'INTEGER4',
4106 'bigint': 'BIGINT',
4107 'float': 'FLOAT',
4108 'double': 'FLOAT8',
4109 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4110 'date': 'ANSIDATE',
4111 'time': 'TIME WITHOUT TIME ZONE',
4112 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
4113 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
4114 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4115 'list:integer': 'NCLOB',
4116 'list:string': 'NCLOB',
4117 'list:reference': 'NCLOB',
4118 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
4119 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4120 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4121 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4122 }
4123
4125 drivers = ('sapdb',)
4126
4127 support_distributed_transaction = False
4128 types = {
4129 'boolean': 'CHAR(1)',
4130 'string': 'VARCHAR(%(length)s)',
4131 'text': 'LONG',
4132 'json': 'LONG',
4133 'password': 'VARCHAR(%(length)s)',
4134 'blob': 'LONG',
4135 'upload': 'VARCHAR(%(length)s)',
4136 'integer': 'INT',
4137 'bigint': 'BIGINT',
4138 'float': 'FLOAT',
4139 'double': 'DOUBLE PRECISION',
4140 'decimal': 'FIXED(%(precision)s,%(scale)s)',
4141 'date': 'DATE',
4142 'time': 'TIME',
4143 'datetime': 'TIMESTAMP',
4144 'id': 'INT PRIMARY KEY',
4145 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4146 'list:integer': 'LONG',
4147 'list:string': 'LONG',
4148 'list:reference': 'LONG',
4149 'big-id': 'BIGINT PRIMARY KEY',
4150 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4151 }
4152
4154 return '%s_id_Seq' % table
4155
4156 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4157 if limitby:
4158 (lmin, lmax) = limitby
4159 if len(sql_w) > 1:
4160 sql_w_row = sql_w + ' AND w_row > %i' % lmin
4161 else:
4162 sql_w_row = 'WHERE w_row > %i' % lmin
4163 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
4164 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4165
4167
4168 self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
4169 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
4170 % (table._tablename, table._id.name, table._sequence_name))
4171 self.execute(query)
4172
4173 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
4174
4175
4176 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4177 credential_decoder=IDENTITY, driver_args={},
4178 adapter_args={}, do_connect=True, after_connection=None):
4179 self.db = db
4180 self.dbengine = "sapdb"
4181 self.uri = uri
4182 if do_connect: self.find_driver(adapter_args,uri)
4183 self.pool_size = pool_size
4184 self.folder = folder
4185 self.db_codec = db_codec
4186 self._after_connection = after_connection
4187 self.find_or_make_work_folder()
4188 ruri = uri.split('://',1)[1]
4189 m = self.REGEX_URI.match(ruri)
4190 if not m:
4191 raise SyntaxError("Invalid URI string in DAL")
4192 user = credential_decoder(m.group('user'))
4193 if not user:
4194 raise SyntaxError('User required')
4195 password = credential_decoder(m.group('password'))
4196 if not password:
4197 password = ''
4198 host = m.group('host')
4199 if not host:
4200 raise SyntaxError('Host name required')
4201 db = m.group('db')
4202 if not db:
4203 raise SyntaxError('Database name required')
4204 def connector(user=user, password=password, database=db,
4205 host=host, driver_args=driver_args):
4206 return self.driver.Connection(user, password, database,
4207 host, **driver_args)
4208 self.connector = connector
4209 if do_connect: self.reconnect()
4210
4212 self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
4213 return long(self.cursor.fetchone()[0])
4214
4216 drivers = ('cubriddb',)
4217
4218 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
4219
4220 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
4221 credential_decoder=IDENTITY, driver_args={},
4222 adapter_args={}, do_connect=True, after_connection=None):
4223 self.db = db
4224 self.dbengine = "cubrid"
4225 self.uri = uri
4226 if do_connect: self.find_driver(adapter_args,uri)
4227 self.pool_size = pool_size
4228 self.folder = folder
4229 self.db_codec = db_codec
4230 self._after_connection = after_connection
4231 self.find_or_make_work_folder()
4232 ruri = uri.split('://',1)[1]
4233 m = self.REGEX_URI.match(ruri)
4234 if not m:
4235 raise SyntaxError(
4236 "Invalid URI string in DAL: %s" % self.uri)
4237 user = credential_decoder(m.group('user'))
4238 if not user:
4239 raise SyntaxError('User required')
4240 password = credential_decoder(m.group('password'))
4241 if not password:
4242 password = ''
4243 host = m.group('host')
4244 if not host:
4245 raise SyntaxError('Host name required')
4246 db = m.group('db')
4247 if not db:
4248 raise SyntaxError('Database name required')
4249 port = int(m.group('port') or '30000')
4250 charset = m.group('charset') or 'utf8'
4251 user = credential_decoder(user)
4252 passwd = credential_decoder(password)
4253 def connector(host=host,port=port,db=db,
4254 user=user,passwd=password,driver_args=driver_args):
4255 return self.driver.connect(host,port,db,user,passwd,**driver_args)
4256 self.connector = connector
4257 if do_connect: self.reconnect()
4258
4260 self.execute('SET FOREIGN_KEY_CHECKS=1;')
4261 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4262
4267
4268 web2py_filesystem = False
4269
4271 return self.db._adapter.escape(obj)
4272
4274 if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'):
4275 raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now")
4276 self.db = db
4277 self.filename = filename
4278 self.mode = mode
4279 if not self.web2py_filesystem:
4280 if db._adapter.dbengine == 'mysql':
4281 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
4282 elif db._adapter.dbengine in ('postgres', 'sqlite'):
4283 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
4284 self.db.executesql(sql)
4285 DatabaseStoredFile.web2py_filesystem = True
4286 self.p=0
4287 self.data = ''
4288 if mode in ('r','rw','a'):
4289 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
4290 % filename
4291 rows = self.db.executesql(query)
4292 if rows:
4293 self.data = rows[0][0]
4294 elif exists(filename):
4295 datafile = open(filename, 'r')
4296 try:
4297 self.data = datafile.read()
4298 finally:
4299 datafile.close()
4300 elif mode in ('r','rw'):
4301 raise RuntimeError("File %s does not exist" % filename)
4302
4303 - def read(self, bytes):
4304 data = self.data[self.p:self.p+bytes]
4305 self.p += len(data)
4306 return data
4307
4309 i = self.data.find('\n',self.p)+1
4310 if i>0:
4311 data, self.p = self.data[self.p:i], i
4312 else:
4313 data, self.p = self.data[self.p:], len(self.data)
4314 return data
4315
4318
4320 if self.db is not None:
4321 self.db.executesql(
4322 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
4323 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
4324 % (self.filename, self.data.replace("'","''"))
4325 self.db.executesql(query)
4326 self.db.commit()
4327 self.db = None
4328
4331
4332 @staticmethod
4334 if exists(filename):
4335 return True
4336 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
4337 try:
4338 if db.executesql(query):
4339 return True
4340 except Exception, e:
4341 if not (db._adapter.isOperationalError(e) or
4342 db._adapter.isProgrammingError(e)):
4343 raise
4344
4345 tb = traceback.format_exc()
4346 LOGGER.error("Could not retrieve %s\n%s" % (filename, tb))
4347 return False
4348
4351
4354
4355 - def file_open(self, filename, mode='rb', lock=True):
4357
4360
4362 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
4363 self.db.executesql(query)
4364 self.db.commit()
4365
4367 uploads_in_blob = True
4368
4369 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
4370
4371 - def __init__(self, db, uri='google:sql://realm:domain/database',
4372 pool_size=0, folder=None, db_codec='UTF-8',
4373 credential_decoder=IDENTITY, driver_args={},
4374 adapter_args={}, do_connect=True, after_connection=None):
4375
4376 self.db = db
4377 self.dbengine = "mysql"
4378 self.uri = uri
4379 self.pool_size = pool_size
4380 self.db_codec = db_codec
4381 self._after_connection = after_connection
4382 if do_connect: self.find_driver(adapter_args, uri)
4383 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
4384 os.sep+'applications'+os.sep,1)[1])
4385 ruri = uri.split("://")[1]
4386 m = self.REGEX_URI.match(ruri)
4387 if not m:
4388 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
4389 instance = credential_decoder(m.group('instance'))
4390 self.dbstring = db = credential_decoder(m.group('db'))
4391 driver_args['instance'] = instance
4392 if not 'charset' in driver_args:
4393 driver_args['charset'] = 'utf8'
4394 self.createdb = createdb = adapter_args.get('createdb',True)
4395 if not createdb:
4396 driver_args['database'] = db
4397 def connector(driver_args=driver_args):
4398 return rdbms.connect(**driver_args)
4399 self.connector = connector
4400 if do_connect: self.reconnect()
4401
4403 if self.createdb:
4404
4405 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
4406 self.execute('USE %s' % self.dbstring)
4407 self.execute("SET FOREIGN_KEY_CHECKS=1;")
4408 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4409
4410 - def execute(self, command, *a, **b):
4412
4414 self.adapter_args = adapter_args
4415 self.driver = "google"
4416
4418 can_select_for_update = False
4419
4420 @staticmethod
4422 if isinstance(obj, str):
4423 return obj.decode('utf8')
4424 elif not isinstance(obj, unicode):
4425 return unicode(obj)
4426 return obj
4427
4429 return table._id > 0
4430
4432 field_is_type = fieldtype.startswith
4433 if isinstance(obj, CALLABLETYPES):
4434 obj = obj()
4435 if isinstance(fieldtype, SQLCustomType):
4436 return fieldtype.encoder(obj)
4437 if isinstance(obj, (Expression, Field)):
4438 raise SyntaxError("non supported on GAE")
4439 if self.dbengine == 'google:datastore':
4440 if isinstance(fieldtype, gae.Property):
4441 return obj
4442 is_string = isinstance(fieldtype,str)
4443 is_list = is_string and field_is_type('list:')
4444 if is_list:
4445 if not obj:
4446 obj = []
4447 if not isinstance(obj, (list, tuple)):
4448 obj = [obj]
4449 if obj == '' and not \
4450 (is_string and fieldtype[:2] in ['st','te', 'pa','up']):
4451 return None
4452 if not obj is None:
4453 if isinstance(obj, list) and not is_list:
4454 obj = [self.represent(o, fieldtype) for o in obj]
4455 elif fieldtype in ('integer','bigint','id'):
4456 obj = long(obj)
4457 elif fieldtype == 'double':
4458 obj = float(obj)
4459 elif is_string and field_is_type('reference'):
4460 if isinstance(obj, (Row, Reference)):
4461 obj = obj['id']
4462 obj = long(obj)
4463 elif fieldtype == 'boolean':
4464 if obj and not str(obj)[0].upper() in '0F':
4465 obj = True
4466 else:
4467 obj = False
4468 elif fieldtype == 'date':
4469 if not isinstance(obj, datetime.date):
4470 (y, m, d) = map(int,str(obj).strip().split('-'))
4471 obj = datetime.date(y, m, d)
4472 elif isinstance(obj,datetime.datetime):
4473 (y, m, d) = (obj.year, obj.month, obj.day)
4474 obj = datetime.date(y, m, d)
4475 elif fieldtype == 'time':
4476 if not isinstance(obj, datetime.time):
4477 time_items = map(int,str(obj).strip().split(':')[:3])
4478 if len(time_items) == 3:
4479 (h, mi, s) = time_items
4480 else:
4481 (h, mi, s) = time_items + [0]
4482 obj = datetime.time(h, mi, s)
4483 elif fieldtype == 'datetime':
4484 if not isinstance(obj, datetime.datetime):
4485 (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
4486 time_items = map(int,str(obj)[11:].strip().split(':')[:3])
4487 while len(time_items)<3:
4488 time_items.append(0)
4489 (h, mi, s) = time_items
4490 obj = datetime.datetime(y, m, d, h, mi, s)
4491 elif fieldtype == 'blob':
4492 pass
4493 elif fieldtype == 'json':
4494 if isinstance(obj, basestring):
4495 obj = self.to_unicode(obj)
4496 if have_serializers:
4497 obj = serializers.loads_json(obj)
4498 elif simplejson:
4499 obj = simplejson.loads(obj)
4500 else:
4501 raise RuntimeError("missing simplejson")
4502 elif is_string and field_is_type('list:string'):
4503 return map(self.to_unicode,obj)
4504 elif is_list:
4505 return map(int,obj)
4506 else:
4507 obj = self.to_unicode(obj)
4508 return obj
4509
4511 return 'insert %s in %s' % (fields, table)
4512
4513 - def _count(self,query,distinct=None):
4514 return 'count %s' % repr(query)
4515
4516 - def _select(self,query,fields,attributes):
4517 return 'select %s where %s' % (repr(fields), repr(query))
4518
4519 - def _delete(self,tablename, query):
4520 return 'delete %s where %s' % (repr(tablename),repr(query))
4521
4522 - def _update(self,tablename,query,fields):
4523 return 'update %s (%s) where %s' % (repr(tablename),
4524 repr(fields),repr(query))
4525
4527 """
4528 remember: no transactions on many NoSQL
4529 """
4530 pass
4531
4533 """
4534 remember: no transactions on many NoSQL
4535 """
4536 pass
4537
4539 """
4540 remember: no transactions on many NoSQL
4541 """
4542 pass
4543
4544
4545
4546 - def OR(self,first,second): raise SyntaxError("Not supported")
4547 - def AND(self,first,second): raise SyntaxError("Not supported")
4548 - def AS(self,first,second): raise SyntaxError("Not supported")
4549 - def ON(self,first,second): raise SyntaxError("Not supported")
4550 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4551 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4552 - def ADD(self,first,second): raise SyntaxError("Not supported")
4553 - def SUB(self,first,second): raise SyntaxError("Not supported")
4554 - def MUL(self,first,second): raise SyntaxError("Not supported")
4555 - def DIV(self,first,second): raise SyntaxError("Not supported")
4556 - def LOWER(self,first): raise SyntaxError("Not supported")
4557 - def UPPER(self,first): raise SyntaxError("Not supported")
4559 - def LENGTH(self, first): raise SyntaxError("Not supported")
4560 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4561 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4562 - def RANDOM(self): raise SyntaxError("Not supported")
4563 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4564 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4565 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4566 - def drop(self,table,mode): raise SyntaxError("Not supported")
4567 - def alias(self,table,alias): raise SyntaxError("Not supported")
4568 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4570 - def prepare(self,key): raise SyntaxError("Not supported")
4573 - def concat_add(self,table): raise SyntaxError("Not supported")
4574 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4576 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4577 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4579 - def lastrowid(self,table): raise SyntaxError("Not supported")
4580 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4581
4582
4583 -class GAEF(object):
4584 - def __init__(self,name,op,value,apply):
4585 self.name=name=='id' and '__key__' or name
4586 self.op=op
4587 self.value=value
4588 self.apply=apply
4590 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4591
4593 uploads_in_blob = True
4594 types = {}
4595
4597 - def file_open(self, filename, mode='rb', lock=True): pass
4599
4600 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
4601
4602 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4603 credential_decoder=IDENTITY, driver_args={},
4604 adapter_args={}, do_connect=True, after_connection=None):
4605 self.types.update({
4606 'boolean': gae.BooleanProperty,
4607 'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)),
4608 'text': gae.TextProperty,
4609 'json': gae.TextProperty,
4610 'password': gae.StringProperty,
4611 'blob': gae.BlobProperty,
4612 'upload': gae.StringProperty,
4613 'integer': gae.IntegerProperty,
4614 'bigint': gae.IntegerProperty,
4615 'float': gae.FloatProperty,
4616 'double': gae.FloatProperty,
4617 'decimal': GAEDecimalProperty,
4618 'date': gae.DateProperty,
4619 'time': gae.TimeProperty,
4620 'datetime': gae.DateTimeProperty,
4621 'id': None,
4622 'reference': gae.IntegerProperty,
4623 'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)),
4624 'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4625 'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4626 })
4627 self.db = db
4628 self.uri = uri
4629 self.dbengine = 'google:datastore'
4630 self.folder = folder
4631 db['_lastsql'] = ''
4632 self.db_codec = 'UTF-8'
4633 self._after_connection = after_connection
4634 self.pool_size = 0
4635 match = self.REGEX_NAMESPACE.match(uri)
4636 if match:
4637 namespace_manager.set_namespace(match.group('namespace'))
4638
4639 - def parse_id(self, value, field_type):
4641
4642 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4643 myfields = {}
4644 for field in table:
4645 if isinstance(polymodel,Table) and field.name in polymodel.fields():
4646 continue
4647 attr = {}
4648 if isinstance(field.custom_qualifier, dict):
4649
4650 attr = field.custom_qualifier
4651 field_type = field.type
4652 if isinstance(field_type, SQLCustomType):
4653 ftype = self.types[field_type.native or field_type.type](**attr)
4654 elif isinstance(field_type, gae.Property):
4655 ftype = field_type
4656 elif field_type.startswith('id'):
4657 continue
4658 elif field_type.startswith('decimal'):
4659 precision, scale = field_type[7:].strip('()').split(',')
4660 precision = int(precision)
4661 scale = int(scale)
4662 ftype = GAEDecimalProperty(precision, scale, **attr)
4663 elif field_type.startswith('reference'):
4664 if field.notnull:
4665 attr = dict(required=True)
4666 referenced = field_type[10:].strip()
4667 ftype = self.types[field_type[:9]](referenced, **attr)
4668 elif field_type.startswith('list:reference'):
4669 if field.notnull:
4670 attr['required'] = True
4671 referenced = field_type[15:].strip()
4672 ftype = self.types[field_type[:14]](**attr)
4673 elif field_type.startswith('list:'):
4674 ftype = self.types[field_type](**attr)
4675 elif not field_type in self.types\
4676 or not self.types[field_type]:
4677 raise SyntaxError('Field: unknown field type: %s' % field_type)
4678 else:
4679 ftype = self.types[field_type](**attr)
4680 myfields[field.name] = ftype
4681 if not polymodel:
4682 table._tableobj = classobj(table._tablename, (gae.Model, ), myfields)
4683 elif polymodel==True:
4684 table._tableobj = classobj(table._tablename, (PolyModel, ), myfields)
4685 elif isinstance(polymodel,Table):
4686 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
4687 else:
4688 raise SyntaxError("polymodel must be None, True, a table or a tablename")
4689 return None
4690
4691 - def expand(self,expression,field_type=None):
4692 if isinstance(expression,Field):
4693 if expression.type in ('text', 'blob', 'json'):
4694 raise SyntaxError('AppEngine does not index by: %s' % expression.type)
4695 return expression.name
4696 elif isinstance(expression, (Expression, Query)):
4697 if not expression.second is None:
4698 return expression.op(expression.first, expression.second)
4699 elif not expression.first is None:
4700 return expression.op(expression.first)
4701 else:
4702 return expression.op()
4703 elif field_type:
4704 return self.represent(expression,field_type)
4705 elif isinstance(expression,(list,tuple)):
4706 return ','.join([self.represent(item,field_type) for item in expression])
4707 else:
4708 return str(expression)
4709
4710
4711 - def AND(self,first,second):
4717
4718 - def EQ(self,first,second=None):
4719 if isinstance(second, Key):
4720 return [GAEF(first.name,'=',second,lambda a,b:a==b)]
4721 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4722
4723 - def NE(self,first,second=None):
4724 if first.type != 'id':
4725 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
4726 else:
4727 if not second is None:
4728 second = Key.from_path(first._tablename, long(second))
4729 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4730
4731 - def LT(self,first,second=None):
4732 if first.type != 'id':
4733 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
4734 else:
4735 second = Key.from_path(first._tablename, long(second))
4736 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4737
4738 - def LE(self,first,second=None):
4739 if first.type != 'id':
4740 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
4741 else:
4742 second = Key.from_path(first._tablename, long(second))
4743 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4744
4745 - def GT(self,first,second=None):
4746 if first.type != 'id' or second==0 or second == '0':
4747 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
4748 else:
4749 second = Key.from_path(first._tablename, long(second))
4750 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4751
4752 - def GE(self,first,second=None):
4753 if first.type != 'id':
4754 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
4755 else:
4756 second = Key.from_path(first._tablename, long(second))
4757 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4758
4761
4762 - def COMMA(self,first,second):
4764
4765 - def BELONGS(self,first,second=None):
4766 if not isinstance(second,(list, tuple)):
4767 raise SyntaxError("Not supported")
4768 if first.type != 'id':
4769 return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
4770 else:
4771 second = [Key.from_path(first._tablename, int(i)) for i in second]
4772 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4773
4774 - def CONTAINS(self,first,second,case_sensitive=False):
4779
4780 - def NOT(self,first):
4781 nops = { self.EQ: self.NE,
4782 self.NE: self.EQ,
4783 self.LT: self.GE,
4784 self.GT: self.LE,
4785 self.LE: self.GT,
4786 self.GE: self.LT}
4787 if not isinstance(first,Query):
4788 raise SyntaxError("Not suported")
4789 nop = nops.get(first.op,None)
4790 if not nop:
4791 raise SyntaxError("Not suported %s" % first.op.__name__)
4792 first.op = nop
4793 return self.expand(first)
4794
4797
4798 - def select_raw(self,query,fields=None,attributes=None):
4799 db = self.db
4800 fields = fields or []
4801 attributes = attributes or {}
4802 args_get = attributes.get
4803 new_fields = []
4804 for item in fields:
4805 if isinstance(item,SQLALL):
4806 new_fields += item._table
4807 else:
4808 new_fields.append(item)
4809 fields = new_fields
4810 if query:
4811 tablename = self.get_table(query)
4812 elif fields:
4813 tablename = fields[0].tablename
4814 query = db._adapter.id_query(fields[0].table)
4815 else:
4816 raise SyntaxError("Unable to determine a tablename")
4817
4818 if query:
4819 if use_common_filters(query):
4820 query = self.common_filter(query,[tablename])
4821
4822
4823 tableobj = db[tablename]._tableobj
4824 filters = self.expand(query)
4825
4826 projection = None
4827 if len(db[tablename].fields) == len(fields):
4828
4829 projection = None
4830 elif args_get('projection') == True:
4831 projection = []
4832 for f in fields:
4833 if f.type in ['text', 'blob', 'json']:
4834 raise SyntaxError(
4835 "text and blob field types not allowed in projection queries")
4836 else:
4837 projection.append(f.name)
4838 elif args_get('filterfields') == True:
4839 projection = []
4840 for f in fields:
4841 projection.append(f.name)
4842
4843
4844
4845 query_projection = [
4846 p for p in projection if \
4847 p != db[tablename]._id.name] if projection and \
4848 args_get('projection') == True\
4849 else None
4850
4851 cursor = None
4852 if isinstance(args_get('reusecursor'), str):
4853 cursor = args_get('reusecursor')
4854 items = gae.Query(tableobj, projection=query_projection,
4855 cursor=cursor)
4856
4857 for filter in filters:
4858 if args_get('projection') == True and \
4859 filter.name in query_projection and \
4860 filter.op in ['=', '<=', '>=']:
4861 raise SyntaxError(
4862 "projection fields cannot have equality filters")
4863 if filter.name=='__key__' and filter.op=='>' and filter.value==0:
4864 continue
4865 elif filter.name=='__key__' and filter.op=='=':
4866 if filter.value==0:
4867 items = []
4868 elif isinstance(filter.value, Key):
4869
4870
4871
4872 item = tableobj.get(filter.value)
4873 items = (item and [item]) or []
4874 else:
4875
4876
4877
4878 item = tableobj.get_by_id(filter.value)
4879 items = (item and [item]) or []
4880 elif isinstance(items,list):
4881 items = [i for i in items if filter.apply(
4882 getattr(item,filter.name),filter.value)]
4883 else:
4884 if filter.name=='__key__' and filter.op != 'in':
4885 items.order('__key__')
4886 items = items.filter('%s %s' % (filter.name,filter.op),
4887 filter.value)
4888 if not isinstance(items,list):
4889 if args_get('left', None):
4890 raise SyntaxError('Set: no left join in appengine')
4891 if args_get('groupby', None):
4892 raise SyntaxError('Set: no groupby in appengine')
4893 orderby = args_get('orderby', False)
4894 if orderby:
4895
4896 if isinstance(orderby, (list, tuple)):
4897 orderby = xorify(orderby)
4898 if isinstance(orderby,Expression):
4899 orderby = self.expand(orderby)
4900 orders = orderby.split(', ')
4901 for order in orders:
4902 order={'-id':'-__key__','id':'__key__'}.get(order,order)
4903 items = items.order(order)
4904 if args_get('limitby', None):
4905 (lmin, lmax) = attributes['limitby']
4906 (limit, offset) = (lmax - lmin, lmin)
4907 rows = items.fetch(limit,offset=offset)
4908
4909
4910 if args_get('reusecursor'):
4911 db['_lastcursor'] = items.cursor()
4912 items = rows
4913 return (items, tablename, projection or db[tablename].fields)
4914
4915 - def select(self,query,fields,attributes):
4916 """
4917 This is the GAE version of select. some notes to consider:
4918 - db['_lastsql'] is not set because there is not SQL statement string
4919 for a GAE query
4920 - 'nativeRef' is a magical fieldname used for self references on GAE
4921 - optional attribute 'projection' when set to True will trigger
4922 use of the GAE projection queries. note that there are rules for
4923 what is accepted imposed by GAE: each field must be indexed,
4924 projection queries cannot contain blob or text fields, and you
4925 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
4926 - optional attribute 'filterfields' when set to True web2py will only
4927 parse the explicitly listed fields into the Rows object, even though
4928 all fields are returned in the query. This can be used to reduce
4929 memory usage in cases where true projection queries are not
4930 usable.
4931 - optional attribute 'reusecursor' allows use of cursor with queries
4932 that have the limitby attribute. Set the attribute to True for the
4933 first query, set it to the value of db['_lastcursor'] to continue
4934 a previous query. The user must save the cursor value between
4935 requests, and the filters must be identical. It is up to the user
4936 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
4937 """
4938
4939 (items, tablename, fields) = self.select_raw(query,fields,attributes)
4940
4941 rows = [[(t==self.db[tablename]._id.name and item) or \
4942 (t=='nativeRef' and item) or getattr(item, t) \
4943 for t in fields] for item in items]
4944 colnames = ['%s.%s' % (tablename, t) for t in fields]
4945 processor = attributes.get('processor',self.parse)
4946 return processor(rows,fields,colnames,False)
4947
4948 - def count(self,query,distinct=None,limit=None):
4949 if distinct:
4950 raise RuntimeError("COUNT DISTINCT not supported")
4951 (items, tablename, fields) = self.select_raw(query)
4952
4953 try:
4954 return len(items)
4955 except TypeError:
4956 return items.count(limit=limit)
4957
4958 - def delete(self,tablename, query):
4959 """
4960 This function was changed on 2010-05-04 because according to
4961 http://code.google.com/p/googleappengine/issues/detail?id=3119
4962 GAE no longer supports deleting more than 1000 records.
4963 """
4964
4965 (items, tablename, fields) = self.select_raw(query)
4966
4967 if not isinstance(items,list):
4968
4969
4970 leftitems = items.fetch(1000, keys_only=True)
4971 counter = 0
4972 while len(leftitems):
4973 counter += len(leftitems)
4974 gae.delete(leftitems)
4975 leftitems = items.fetch(1000, keys_only=True)
4976 else:
4977 counter = len(items)
4978 gae.delete(items)
4979 return counter
4980
4981 - def update(self,tablename,query,update_fields):
4982
4983 (items, tablename, fields) = self.select_raw(query)
4984 counter = 0
4985 for item in items:
4986 for field, value in update_fields:
4987 setattr(item, field.name, self.represent(value,field.type))
4988 item.put()
4989 counter += 1
4990 LOGGER.info(str(counter))
4991 return counter
4992
4993 - def insert(self,table,fields):
4994 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
4995
4996 tmp = table._tableobj(**dfields)
4997 tmp.put()
4998 rid = Reference(tmp.key().id())
4999 (rid._table, rid._record, rid._gaekey) = (table, None, tmp.key())
5000 return rid
5001
5003 parsed_items = []
5004 for item in items:
5005 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
5006 parsed_items.append(table._tableobj(**dfields))
5007 gae.put(parsed_items)
5008 return True
5009
5011 return uuid.UUID(uuidv).int
5012
5014 return str(uuid.UUID(int=n))
5015
5017 drivers = ('couchdb',)
5018
5019 uploads_in_blob = True
5020 types = {
5021 'boolean': bool,
5022 'string': str,
5023 'text': str,
5024 'json': str,
5025 'password': str,
5026 'blob': str,
5027 'upload': str,
5028 'integer': long,
5029 'bigint': long,
5030 'float': float,
5031 'double': float,
5032 'date': datetime.date,
5033 'time': datetime.time,
5034 'datetime': datetime.datetime,
5035 'id': long,
5036 'reference': long,
5037 'list:string': list,
5038 'list:integer': list,
5039 'list:reference': list,
5040 }
5041
5043 - def file_open(self, filename, mode='rb', lock=True): pass
5045
5046 - def expand(self,expression,field_type=None):
5047 if isinstance(expression,Field):
5048 if expression.type=='id':
5049 return "%s._id" % expression.tablename
5050 return BaseAdapter.expand(self,expression,field_type)
5051
5052 - def AND(self,first,second):
5054
5055 - def OR(self,first,second):
5057
5058 - def EQ(self,first,second):
5062
5063 - def NE(self,first,second):
5067
5068 - def COMMA(self,first,second):
5070
5072 value = NoSQLAdapter.represent(self, obj, fieldtype)
5073 if fieldtype=='id':
5074 return repr(str(long(value)))
5075 elif fieldtype in ('date','time','datetime','boolean'):
5076 return serializers.json(value)
5077 return repr(not isinstance(value,unicode) and value \
5078 or value and value.encode('utf8'))
5079
5080 - def __init__(self,db,uri='couchdb://127.0.0.1:5984',
5081 pool_size=0,folder=None,db_codec ='UTF-8',
5082 credential_decoder=IDENTITY, driver_args={},
5083 adapter_args={}, do_connect=True, after_connection=None):
5084 self.db = db
5085 self.uri = uri
5086 if do_connect: self.find_driver(adapter_args)
5087 self.dbengine = 'couchdb'
5088 self.folder = folder
5089 db['_lastsql'] = ''
5090 self.db_codec = 'UTF-8'
5091 self._after_connection = after_connection
5092 self.pool_size = pool_size
5093
5094 url='http://'+uri[10:]
5095 def connector(url=url,driver_args=driver_args):
5096 return self.driver.Server(url,**driver_args)
5097 self.reconnect(connector,cursor=False)
5098
5099 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
5100 if migrate:
5101 try:
5102 self.connection.create(table._tablename)
5103 except:
5104 pass
5105
5106 - def insert(self,table,fields):
5113
5114 - def _select(self,query,fields,attributes):
5115 if not isinstance(query,Query):
5116 raise SyntaxError("Not Supported")
5117 for key in set(attributes.keys())-SELECT_ARGS:
5118 raise SyntaxError('invalid select attribute: %s' % key)
5119 new_fields=[]
5120 for item in fields:
5121 if isinstance(item,SQLALL):
5122 new_fields += item._table
5123 else:
5124 new_fields.append(item)
5125 def uid(fd):
5126 return fd=='id' and '_id' or fd
5127 def get(row,fd):
5128 return fd=='id' and long(row['_id']) or row.get(fd,None)
5129 fields = new_fields
5130 tablename = self.get_table(query)
5131 fieldnames = [f.name for f in (fields or self.db[tablename])]
5132 colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
5133 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
5134 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
5135 dict(t=tablename,
5136 query=self.expand(query),
5137 order='%s._id' % tablename,
5138 fields=fields)
5139 return fn, colnames
5140
5141 - def select(self,query,fields,attributes):
5142 if not isinstance(query,Query):
5143 raise SyntaxError("Not Supported")
5144 fn, colnames = self._select(query,fields,attributes)
5145 tablename = colnames[0].split('.')[0]
5146 ctable = self.connection[tablename]
5147 rows = [cols['value'] for cols in ctable.query(fn)]
5148 processor = attributes.get('processor',self.parse)
5149 return processor(rows,fields,colnames,False)
5150
5151 - def delete(self,tablename,query):
5152 if not isinstance(query,Query):
5153 raise SyntaxError("Not Supported")
5154 if query.first.type=='id' and query.op==self.EQ:
5155 id = query.second
5156 tablename = query.first.tablename
5157 assert(tablename == query.first.tablename)
5158 ctable = self.connection[tablename]
5159 try:
5160 del ctable[str(id)]
5161 return 1
5162 except couchdb.http.ResourceNotFound:
5163 return 0
5164 else:
5165 tablename = self.get_table(query)
5166 rows = self.select(query,[self.db[tablename]._id],{})
5167 ctable = self.connection[tablename]
5168 for row in rows:
5169 del ctable[str(row.id)]
5170 return len(rows)
5171
5172 - def update(self,tablename,query,fields):
5173 if not isinstance(query,Query):
5174 raise SyntaxError("Not Supported")
5175 if query.first.type=='id' and query.op==self.EQ:
5176 id = query.second
5177 tablename = query.first.tablename
5178 ctable = self.connection[tablename]
5179 try:
5180 doc = ctable[str(id)]
5181 for key,value in fields:
5182 doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
5183 ctable.save(doc)
5184 return 1
5185 except couchdb.http.ResourceNotFound:
5186 return 0
5187 else:
5188 tablename = self.get_table(query)
5189 rows = self.select(query,[self.db[tablename]._id],{})
5190 ctable = self.connection[tablename]
5191 table = self.db[tablename]
5192 for row in rows:
5193 doc = ctable[str(row.id)]
5194 for key,value in fields:
5195 doc[key.name] = self.represent(value,table[key.name].type)
5196 ctable.save(doc)
5197 return len(rows)
5198
5199 - def count(self,query,distinct=None):
5200 if distinct:
5201 raise RuntimeError("COUNT DISTINCT not supported")
5202 if not isinstance(query,Query):
5203 raise SyntaxError("Not Supported")
5204 tablename = self.get_table(query)
5205 rows = self.select(query,[self.db[tablename]._id],{})
5206 return len(rows)
5207
5209 """
5210 validates that the given text is clean: only contains [0-9a-zA-Z_]
5211 """
5212 if not REGEX_ALPHANUMERIC.match(text):
5213 raise SyntaxError('invalid table or field name: %s' % text)
5214 return text
5215
5217 native_json = True
5218 drivers = ('pymongo',)
5219
5220 uploads_in_blob = True
5221
5222 types = {
5223 'boolean': bool,
5224 'string': str,
5225 'text': str,
5226 'json': str,
5227 'password': str,
5228 'blob': str,
5229 'upload': str,
5230 'integer': long,
5231 'bigint': long,
5232 'float': float,
5233 'double': float,
5234 'date': datetime.date,
5235 'time': datetime.time,
5236 'datetime': datetime.datetime,
5237 'id': long,
5238 'reference': long,
5239 'list:string': list,
5240 'list:integer': list,
5241 'list:reference': list,
5242 }
5243
5244 error_messages = {"javascript_needed": "This must yet be replaced" +
5245 " with javascript in order to work."}
5246
5247 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
5248 pool_size=0, folder=None, db_codec ='UTF-8',
5249 credential_decoder=IDENTITY, driver_args={},
5250 adapter_args={}, do_connect=True, after_connection=None):
5251
5252 self.db = db
5253 self.uri = uri
5254 if do_connect: self.find_driver(adapter_args)
5255 import random
5256 from bson.objectid import ObjectId
5257 from bson.son import SON
5258 import pymongo.uri_parser
5259
5260 m = pymongo.uri_parser.parse_uri(uri)
5261
5262 self.SON = SON
5263 self.ObjectId = ObjectId
5264 self.random = random
5265
5266 self.dbengine = 'mongodb'
5267 self.folder = folder
5268 db['_lastsql'] = ''
5269 self.db_codec = 'UTF-8'
5270 self._after_connection = after_connection
5271 self.pool_size = pool_size
5272
5273
5274 self.minimumreplication = adapter_args.get('minimumreplication',0)
5275
5276
5277
5278
5279 self.safe = adapter_args.get('safe',True)
5280
5281 if isinstance(m,tuple):
5282 m = {"database" : m[1]}
5283 if m.get('database')==None:
5284 raise SyntaxError("Database is required!")
5285
5286 def connector(uri=self.uri,m=m):
5287
5288 if hasattr(self.driver, "MongoClient"):
5289 Connection = self.driver.MongoClient
5290 else:
5291 Connection = self.driver.Connection
5292 return Connection(uri)[m.get('database')]
5293
5294 self.reconnect(connector,cursor=False)
5295
5297 """ Convert input to a valid Mongodb ObjectId instance
5298
5299 self.object_id("<random>") -> ObjectId (not unique) instance """
5300 if not arg:
5301 arg = 0
5302 if isinstance(arg, basestring):
5303
5304 rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
5305 if arg.isdigit() and (not rawhex):
5306 arg = int(arg)
5307 elif arg == "<random>":
5308 arg = int("0x%sL" % \
5309 "".join([self.random.choice("0123456789abcdef") \
5310 for x in range(24)]), 0)
5311 elif arg.isalnum():
5312 if not arg.startswith("0x"):
5313 arg = "0x%s" % arg
5314 try:
5315 arg = int(arg, 0)
5316 except ValueError, e:
5317 raise ValueError(
5318 "invalid objectid argument string: %s" % e)
5319 else:
5320 raise ValueError("Invalid objectid argument string. " +
5321 "Requires an integer or base 16 value")
5322 elif isinstance(arg, self.ObjectId):
5323 return arg
5324
5325 if not isinstance(arg, (int, long)):
5326 raise TypeError("object_id argument must be of type " +
5327 "ObjectId or an objectid representable integer")
5328 if arg == 0:
5329 hexvalue = "".zfill(24)
5330 else:
5331 hexvalue = hex(arg)[2:].replace("L", "")
5332 return self.ObjectId(hexvalue)
5333
5335
5336 if isinstance(value, self.ObjectId):
5337 value = long(str(value), 16)
5338 return super(MongoDBAdapter,
5339 self).parse_reference(value, field_type)
5340
5341 - def parse_id(self, value, field_type):
5342 if isinstance(value, self.ObjectId):
5343 value = long(str(value), 16)
5344 return super(MongoDBAdapter,
5345 self).parse_id(value, field_type)
5346
5348
5349 if isinstance(obj, self.ObjectId):
5350 value = obj
5351 else:
5352 value = NoSQLAdapter.represent(self, obj, fieldtype)
5353
5354 if fieldtype =='date':
5355 if value == None:
5356 return value
5357
5358 t = datetime.time(0, 0, 0)
5359
5360
5361 return datetime.datetime.combine(value, t)
5362 elif fieldtype == 'time':
5363 if value == None:
5364 return value
5365
5366 d = datetime.date(2000, 1, 1)
5367
5368
5369 return datetime.datetime.combine(d, value)
5370 elif fieldtype == "blob":
5371 from bson import Binary
5372 if not isinstance(value, Binary):
5373 return Binary(value)
5374 return value
5375 elif (isinstance(fieldtype, basestring) and
5376 fieldtype.startswith('list:')):
5377 if fieldtype.startswith('list:reference'):
5378 newval = []
5379 for v in value:
5380 newval.append(self.object_id(v))
5381 return newval
5382 return value
5383 elif ((isinstance(fieldtype, basestring) and
5384 fieldtype.startswith("reference")) or
5385 (isinstance(fieldtype, Table)) or fieldtype=="id"):
5386 value = self.object_id(value)
5387 return value
5388
5389 - def create_table(self, table, migrate=True, fake_migrate=False,
5390 polymodel=None, isCapped=False):
5391 if isCapped:
5392 raise RuntimeError("Not implemented")
5393
5394 - def count(self, query, distinct=None, snapshot=True):
5395 if distinct:
5396 raise RuntimeError("COUNT DISTINCT not supported")
5397 if not isinstance(query,Query):
5398 raise SyntaxError("Not Supported")
5399 tablename = self.get_table(query)
5400 return long(self.select(query,[self.db[tablename]._id], {},
5401 count=True,snapshot=snapshot)['count'])
5402
5403
5404
5405
5406
5407 - def expand(self, expression, field_type=None):
5408 if isinstance(expression, Query):
5409
5410
5411
5412
5413
5414 if isinstance(expression.first,Field) and \
5415 ((expression.first.type == 'id') or \
5416 ("reference" in expression.first.type)):
5417 if expression.first.type == 'id':
5418 expression.first.name = '_id'
5419
5420 if isinstance(expression.second, (tuple, list, set)):
5421 expression.second = [self.object_id(item) for
5422 item in expression.second]
5423 else:
5424 expression.second = self.object_id(expression.second)
5425 result = expression.op(expression.first, expression.second)
5426
5427 if isinstance(expression, Field):
5428 if expression.type=='id':
5429 result = "_id"
5430 else:
5431 result = expression.name
5432 elif isinstance(expression, (Expression, Query)):
5433 if not expression.second is None:
5434 result = expression.op(expression.first, expression.second)
5435 elif not expression.first is None:
5436 result = expression.op(expression.first)
5437 elif not isinstance(expression.op, str):
5438 result = expression.op()
5439 else:
5440 result = expression.op
5441 elif field_type:
5442 result = self.represent(expression,field_type)
5443 elif isinstance(expression,(list,tuple)):
5444 result = ','.join(self.represent(item,field_type) for
5445 item in expression)
5446 else:
5447 result = expression
5448 return result
5449
5450 - def drop(self, table, mode=''):
5453
5454 - def truncate(self, table, mode, safe=None):
5455 if safe == None:
5456 safe=self.safe
5457 ctable = self.connection[table._tablename]
5458 ctable.remove(None, safe=True)
5459
5460 - def _select(self, query, fields, attributes):
5461 if 'for_update' in attributes:
5462 logging.warn('mongodb does not support for_update')
5463 for key in set(attributes.keys())-set(('limitby',
5464 'orderby','for_update')):
5465 if attributes[key]!=None:
5466 logging.warn('select attribute not implemented: %s' % key)
5467
5468 new_fields=[]
5469 mongosort_list = []
5470
5471
5472 orderby = attributes.get('orderby', False)
5473 limitby = attributes.get('limitby', False)
5474
5475 if orderby:
5476 if isinstance(orderby, (list, tuple)):
5477 orderby = xorify(orderby)
5478
5479
5480 for f in self.expand(orderby).split(','):
5481 if f.startswith('-'):
5482 mongosort_list.append((f[1:], -1))
5483 else:
5484 mongosort_list.append((f, 1))
5485 if limitby:
5486 limitby_skip, limitby_limit = limitby[0], int(limitby[1])
5487 else:
5488 limitby_skip = limitby_limit = 0
5489
5490 mongofields_dict = self.SON()
5491 mongoqry_dict = {}
5492 for item in fields:
5493 if isinstance(item, SQLALL):
5494 new_fields += item._table
5495 else:
5496 new_fields.append(item)
5497 fields = new_fields
5498 if isinstance(query,Query):
5499 tablename = self.get_table(query)
5500 elif len(fields) != 0:
5501 tablename = fields[0].tablename
5502 else:
5503 raise SyntaxError("The table name could not be found in " +
5504 "the query nor from the select statement.")
5505 mongoqry_dict = self.expand(query)
5506 fields = fields or self.db[tablename]
5507 for field in fields:
5508 mongofields_dict[field.name] = 1
5509
5510 return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5511 limitby_limit, limitby_skip
5512
5513 - def select(self, query, fields, attributes, count=False,
5514 snapshot=False):
5515
5516 tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5517 limitby_limit, limitby_skip = self._select(query, fields, attributes)
5518 ctable = self.connection[tablename]
5519
5520 if count:
5521 return {'count' : ctable.find(
5522 mongoqry_dict, mongofields_dict,
5523 skip=limitby_skip, limit=limitby_limit,
5524 sort=mongosort_list, snapshot=snapshot).count()}
5525 else:
5526
5527 mongo_list_dicts = ctable.find(mongoqry_dict,
5528 mongofields_dict, skip=limitby_skip,
5529 limit=limitby_limit, sort=mongosort_list,
5530 snapshot=snapshot)
5531 rows = []
5532
5533
5534 colnames = []
5535 newnames = []
5536 for field in fields:
5537 colname = str(field)
5538 colnames.append(colname)
5539 tablename, fieldname = colname.split(".")
5540 if fieldname == "_id":
5541
5542 field.name = "id"
5543 newnames.append(".".join((tablename, field.name)))
5544
5545 for record in mongo_list_dicts:
5546 row=[]
5547 for colname in colnames:
5548 tablename, fieldname = colname.split(".")
5549
5550
5551 if fieldname == "id": fieldname = "_id"
5552 if fieldname in record:
5553 value = record[fieldname]
5554 else:
5555 value = None
5556 row.append(value)
5557 rows.append(row)
5558
5559 processor = attributes.get('processor', self.parse)
5560 result = processor(rows, fields, newnames, False)
5561 return result
5562
5563 - def _insert(self, table, fields):
5571
5572
5573
5574
5575 - def insert(self, table, fields, safe=None):
5582
5583
5584 - def _update(self, tablename, query, fields):
5585 if not isinstance(query, Query):
5586 raise SyntaxError("Not Supported")
5587 filter = None
5588 if query:
5589 filter = self.expand(query)
5590
5591 modify = {'$set': dict((k.name, self.represent(v, k.type)) for
5592 k, v in fields if (not k.name in ("_id", "id")))}
5593 return modify, filter
5594
5595 - def update(self, tablename, query, fields, safe=None):
5596 if safe == None:
5597 safe = self.safe
5598
5599
5600 if not isinstance(query, Query):
5601 raise RuntimeError("Not implemented")
5602 amount = self.count(query, False)
5603 modify, filter = self._update(tablename, query, fields)
5604 try:
5605 result = self.connection[tablename].update(filter,
5606 modify, multi=True, safe=safe)
5607 if safe:
5608 try:
5609
5610 return result["n"]
5611 except (KeyError, AttributeError, TypeError):
5612 return amount
5613 else:
5614 return amount
5615 except Exception, e:
5616
5617 raise RuntimeError("uncaught exception when updating rows: %s" % e)
5618
5619 - def _delete(self, tablename, query):
5620 if not isinstance(query, Query):
5621 raise RuntimeError("query type %s is not supported" % \
5622 type(query))
5623 return self.expand(query)
5624
5625 - def delete(self, tablename, query, safe=None):
5626 if safe is None:
5627 safe = self.safe
5628 amount = 0
5629 amount = self.count(query, False)
5630 filter = self._delete(tablename, query)
5631 self.connection[tablename].remove(filter, safe=safe)
5632 return amount
5633
5635 return [self.insert(table,item) for item in items]
5636
5637
5641
5642
5643 - def NOT(self, first):
5644 result = {}
5645 result["$not"] = self.expand(first)
5646 return result
5647
5648 - def AND(self,first,second):
5653
5654 - def OR(self,first,second):
5655
5656 result = {}
5657 f = self.expand(first)
5658 s = self.expand(second)
5659 result['$or'] = [f,s]
5660 return result
5661
5662 - def BELONGS(self, first, second):
5663 if isinstance(second, str):
5664 return {self.expand(first) : {"$in" : [ second[:-1]]} }
5665 elif second==[] or second==() or second==set():
5666 return {1:0}
5667 items = [self.expand(item, first.type) for item in second]
5668 return {self.expand(first) : {"$in" : items} }
5669
5670 - def EQ(self,first,second=None):
5671 result = {}
5672 result[self.expand(first)] = self.expand(second)
5673 return result
5674
5675 - def NE(self, first, second=None):
5676 result = {}
5677 result[self.expand(first)] = {'$ne': self.expand(second)}
5678 return result
5679
5680 - def LT(self,first,second=None):
5681 if second is None:
5682 raise RuntimeError("Cannot compare %s < None" % first)
5683 result = {}
5684 result[self.expand(first)] = {'$lt': self.expand(second)}
5685 return result
5686
5687 - def LE(self,first,second=None):
5688 if second is None:
5689 raise RuntimeError("Cannot compare %s <= None" % first)
5690 result = {}
5691 result[self.expand(first)] = {'$lte': self.expand(second)}
5692 return result
5693
5694 - def GT(self,first,second):
5695 result = {}
5696 result[self.expand(first)] = {'$gt': self.expand(second)}
5697 return result
5698
5699 - def GE(self,first,second=None):
5700 if second is None:
5701 raise RuntimeError("Cannot compare %s >= None" % first)
5702 result = {}
5703 result[self.expand(first)] = {'$gte': self.expand(second)}
5704 return result
5705
5706 - def ADD(self, first, second):
5710
5711 - def SUB(self, first, second):
5715
5716 - def MUL(self, first, second):
5720
5721 - def DIV(self, first, second):
5725
5726 - def MOD(self, first, second):
5730
5731 - def AS(self, first, second):
5732 raise NotImplementedError(self.error_messages["javascript_needed"])
5733 return '%s AS %s' % (self.expand(first), second)
5734
5735
5736
5737
5738 - def ON(self, first, second):
5739 raise NotImplementedError("This is not possible in NoSQL" +
5740 " but can be simulated with a wrapper.")
5741 return '%s ON %s' % (self.expand(first), self.expand(second))
5742
5743
5744
5745
5746 - def COMMA(self, first, second):
5748
5749 - def LIKE(self, first, second):
5753
5755
5756 return {self.expand(first): ('/^%s/' % \
5757 self.expand(second, 'string'))}
5758
5760
5761 return {self.expand(first): ('/%s^/' % \
5762 self.expand(second, 'string'))}
5763
5764 - def CONTAINS(self, first, second, case_sensitive=False):
5765
5766
5767
5768 val = second if isinstance(second,self.ObjectId) else \
5769 {'$regex':".*" + re.escape(self.expand(second, 'string')) + ".*"}
5770 return {self.expand(first) : val}
5771
5772 - def LIKE(self, first, second):
5777
5778
5780
5781 import re
5782 return {self.expand(first): {'$regex' : '^' +
5783 re.escape(self.expand(second,
5784 'string'))}}
5785
5786
5788
5789
5790
5791
5792 import re
5793 return {self.expand(first): {'$regex': \
5794 re.escape(self.expand(second, 'string')) + '$'}}
5795
5796
5797 - def CONTAINS(self, first, second, case_sensitive=False):
5798
5799
5800
5801
5802 return {self.expand(first) : {'$regex': \
5803 ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
5804
5807 drivers = ('imaplib',)
5808
5809 """ IMAP server adapter
5810
5811 This class is intended as an interface with
5812 email IMAP servers to perform simple queries in the
5813 web2py DAL query syntax, so email read, search and
5814 other related IMAP mail services (as those implemented
5815 by brands like Google(r), and Yahoo!(r)
5816 can be managed from web2py applications.
5817
5818 The code uses examples by Yuji Tomita on this post:
5819 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
5820 and is based in docs for Python imaplib, python email
5821 and email IETF's (i.e. RFC2060 and RFC3501)
5822
5823 This adapter was tested with a small set of operations with Gmail(r). Other
5824 services requests could raise command syntax and response data issues.
5825
5826 It creates its table and field names "statically",
5827 meaning that the developer should leave the table and field
5828 definitions to the DAL instance by calling the adapter's
5829 .define_tables() method. The tables are defined with the
5830 IMAP server mailbox list information.
5831
5832 .define_tables() returns a dictionary mapping dal tablenames
5833 to the server mailbox names with the following structure:
5834
5835 {<tablename>: str <server mailbox name>}
5836
5837 Here is a list of supported fields:
5838
5839 Field Type Description
5840 ################################################################
5841 uid string
5842 answered boolean Flag
5843 created date
5844 content list:string A list of dict text or html parts
5845 to string
5846 cc string
5847 bcc string
5848 size integer the amount of octets of the message*
5849 deleted boolean Flag
5850 draft boolean Flag
5851 flagged boolean Flag
5852 sender string
5853 recent boolean Flag
5854 seen boolean Flag
5855 subject string
5856 mime string The mime header declaration
5857 email string The complete RFC822 message**
5858 attachments <type list> Each non text part as dict
5859 encoding string The main detected encoding
5860
5861 *At the application side it is measured as the length of the RFC822
5862 message string
5863
5864 WARNING: As row id's are mapped to email sequence numbers,
5865 make sure your imap client web2py app does not delete messages
5866 during select or update actions, to prevent
5867 updating or deleting different messages.
5868 Sequence numbers change whenever the mailbox is updated.
5869 To avoid this sequence numbers issues, it is recommended the use
5870 of uid fields in query references (although the update and delete
5871 in separate actions rule still applies).
5872
5873 # This is the code recommended to start imap support
5874 # at the app's model:
5875
5876 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
5877 imapdb.define_tables()
5878
5879 Here is an (incomplete) list of possible imap commands:
5880
5881 # Count today's unseen messages
5882 # smaller than 6000 octets from the
5883 # inbox mailbox
5884
5885 q = imapdb.INBOX.seen == False
5886 q &= imapdb.INBOX.created == datetime.date.today()
5887 q &= imapdb.INBOX.size < 6000
5888 unread = imapdb(q).count()
5889
5890 # Fetch last query messages
5891 rows = imapdb(q).select()
5892
5893 # it is also possible to filter query select results with limitby and
5894 # sequences of mailbox fields
5895
5896 set.select(<fields sequence>, limitby=(<int>, <int>))
5897
5898 # Mark last query messages as seen
5899 messages = [row.uid for row in rows]
5900 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
5901
5902 # Delete messages in the imap database that have mails from mr. Gumby
5903
5904 deleted = 0
5905 for mailbox in imapdb.tables
5906 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
5907
5908 # It is possible also to mark messages for deletion instead of ereasing them
5909 # directly with set.update(deleted=True)
5910
5911
5912 # This object give access
5913 # to the adapter auto mailbox
5914 # mapped names (which native
5915 # mailbox has what table name)
5916
5917 imapdb.mailboxes <dict> # tablename, server native name pairs
5918
5919 # To retrieve a table native mailbox name use:
5920 imapdb.<table>.mailbox
5921
5922 ### New features v2.4.1:
5923
5924 # Declare mailboxes statically with tablename, name pairs
5925 # This avoids the extra server names retrieval
5926
5927 imapdb.define_tables({"inbox": "INBOX"})
5928
5929 # Selects without content/attachments/email columns will only
5930 # fetch header and flags
5931
5932 imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject)
5933 """
5934
5935 types = {
5936 'string': str,
5937 'text': str,
5938 'date': datetime.date,
5939 'datetime': datetime.datetime,
5940 'id': long,
5941 'boolean': bool,
5942 'integer': int,
5943 'bigint': long,
5944 'blob': str,
5945 'list:string': str,
5946 }
5947
5948 dbengine = 'imap'
5949
5950 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')
5951
5952 - def __init__(self,
5953 db,
5954 uri,
5955 pool_size=0,
5956 folder=None,
5957 db_codec ='UTF-8',
5958 credential_decoder=IDENTITY,
5959 driver_args={},
5960 adapter_args={},
5961 do_connect=True,
5962 after_connection=None):
5963
5964
5965
5966
5967 self.db = db
5968 self.uri = uri
5969 if do_connect: self.find_driver(adapter_args)
5970 self.pool_size=pool_size
5971 self.folder = folder
5972 self.db_codec = db_codec
5973 self._after_connection = after_connection
5974 self.credential_decoder = credential_decoder
5975 self.driver_args = driver_args
5976 self.adapter_args = adapter_args
5977 self.mailbox_size = None
5978 self.static_names = None
5979 self.charset = sys.getfilesystemencoding()
5980
5981 self.imap4 = None
5982 uri = uri.split("://")[1]
5983
5984 """ MESSAGE is an identifier for sequence number"""
5985
5986 self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft',
5987 'flagged': '\\Flagged', 'recent': '\\Recent',
5988 'seen': '\\Seen', 'answered': '\\Answered'}
5989 self.search_fields = {
5990 'id': 'MESSAGE', 'created': 'DATE',
5991 'uid': 'UID', 'sender': 'FROM',
5992 'to': 'TO', 'cc': 'CC',
5993 'bcc': 'BCC', 'content': 'TEXT',
5994 'size': 'SIZE', 'deleted': '\\Deleted',
5995 'draft': '\\Draft', 'flagged': '\\Flagged',
5996 'recent': '\\Recent', 'seen': '\\Seen',
5997 'subject': 'SUBJECT', 'answered': '\\Answered',
5998 'mime': None, 'email': None,
5999 'attachments': None
6000 }
6001
6002 db['_lastsql'] = ''
6003
6004 m = self.REGEX_URI.match(uri)
6005 user = m.group('user')
6006 password = m.group('password')
6007 host = m.group('host')
6008 port = int(m.group('port'))
6009 over_ssl = False
6010 if port==993:
6011 over_ssl = True
6012
6013 driver_args.update(host=host,port=port, password=password, user=user)
6014 def connector(driver_args=driver_args):
6015
6016
6017 if over_ssl:
6018 self.imap4 = self.driver.IMAP4_SSL
6019 else:
6020 self.imap4 = self.driver.IMAP4
6021 connection = self.imap4(driver_args["host"], driver_args["port"])
6022 data = connection.login(driver_args["user"], driver_args["password"])
6023
6024
6025 connection.mailbox_names = None
6026
6027
6028 connection.cursor = lambda : True
6029
6030 return connection
6031
6032 self.db.define_tables = self.define_tables
6033 self.connector = connector
6034 if do_connect: self.reconnect()
6035
6080
6082 last_message = None
6083
6084
6085 if not isinstance(self.connection.mailbox_names, dict):
6086 self.get_mailboxes()
6087 try:
6088 result = self.connection.select(self.connection.mailbox_names[tablename])
6089 last_message = int(result[1][0])
6090 except (IndexError, ValueError, TypeError, KeyError):
6091 e = sys.exc_info()[1]
6092 LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e))
6093 return last_message
6094
6096 if not isinstance(self.connection.mailbox_names, dict):
6097 self.get_mailboxes()
6098
6099
6100 last_message = self.get_last_message(tablename)
6101 result, data = self.connection.uid("search", None, "(ALL)")
6102 uid_list = data[0].strip().split()
6103 if len(uid_list) <= 0:
6104 return None
6105 else:
6106 return (uid_list[0], uid_list[-1])
6107
6109 if add is None:
6110 add = datetime.timedelta()
6111 """ Convert a date object to a string
6112 with d-Mon-Y style for IMAP or the inverse
6113 case
6114
6115 add <timedelta> adds to the date object
6116 """
6117 months = [None, "JAN","FEB","MAR","APR","MAY","JUN",
6118 "JUL", "AUG","SEP","OCT","NOV","DEC"]
6119 if isinstance(date, basestring):
6120
6121 try:
6122 dayname, datestring = date.split(",")
6123 date_list = datestring.strip().split()
6124 year = int(date_list[2])
6125 month = months.index(date_list[1].upper())
6126 day = int(date_list[0])
6127 hms = map(int, date_list[3].split(":"))
6128 return datetime.datetime(year, month, day,
6129 hms[0], hms[1], hms[2]) + add
6130 except (ValueError, AttributeError, IndexError), e:
6131 LOGGER.error("Could not parse date text: %s. %s" %
6132 (date, e))
6133 return None
6134 elif isinstance(date, (datetime.datetime, datetime.date)):
6135 return (date + add).strftime("%d-%b-%Y")
6136 else:
6137 return None
6138
6139 @staticmethod
6141 from email.header import decode_header
6142 text, encoding = decode_header(f)[0]
6143 if encoding:
6144 text = text.decode(encoding).encode('utf-8')
6145 return text
6146
6147 - def encode_text(self, text, charset, errors="replace"):
6148 """ convert text for mail to unicode"""
6149 if text is None:
6150 text = ""
6151 else:
6152 if isinstance(text, str):
6153 if charset is None:
6154 text = unicode(text, "utf-8", errors)
6155 else:
6156 text = unicode(text, charset, errors)
6157 else:
6158 raise Exception("Unsupported mail text type %s" % type(text))
6159 return text.encode("utf-8")
6160
6162 charset = message.get_content_charset()
6163 return charset
6164
6166 """ Query the mail database for mailbox names """
6167 if self.static_names:
6168
6169 self.connection.mailbox_names = self.static_names
6170 return self.static_names.keys()
6171
6172 mailboxes_list = self.connection.list()
6173 self.connection.mailbox_names = dict()
6174 mailboxes = list()
6175 x = 0
6176 for item in mailboxes_list[1]:
6177 x = x + 1
6178 item = item.strip()
6179 if not "NOSELECT" in item.upper():
6180 sub_items = item.split("\"")
6181 sub_items = [sub_item for sub_item in sub_items \
6182 if len(sub_item.strip()) > 0]
6183
6184 mailbox = sub_items[-1]
6185
6186
6187 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
6188 mailboxes.append(mailbox_name)
6189 self.connection.mailbox_names[mailbox_name] = mailbox
6190
6191 return mailboxes
6192
6194 nofield = True
6195 tablename = None
6196 attr = query
6197 while nofield:
6198 if hasattr(attr, "first"):
6199 attr = attr.first
6200 if isinstance(attr, Field):
6201 return attr.tablename
6202 elif isinstance(attr, Query):
6203 pass
6204 else:
6205 return None
6206 else:
6207 return None
6208 return tablename
6209
6211 if self.search_fields.get(flag, None) in self.flags.values():
6212 return True
6213 else:
6214 return False
6215
6217 """
6218 Auto create common IMAP fileds
6219
6220 This function creates fields definitions "statically"
6221 meaning that custom fields as in other adapters should
6222 not be supported and definitions handled on a service/mode
6223 basis (local syntax for Gmail(r), Ymail(r)
6224
6225 Returns a dictionary with tablename, server native mailbox name
6226 pairs.
6227 """
6228 if mailbox_names:
6229
6230 self.static_names = mailbox_names
6231 else:
6232 self.static_names = None
6233 if not isinstance(self.connection.mailbox_names, dict):
6234 self.get_mailboxes()
6235
6236 names = self.connection.mailbox_names.keys()
6237
6238 for name in names:
6239 self.db.define_table("%s" % name,
6240 Field("uid", "string", writable=False),
6241 Field("answered", "boolean"),
6242 Field("created", "datetime", writable=False),
6243 Field("content", list, writable=False),
6244 Field("to", "string", writable=False),
6245 Field("cc", "string", writable=False),
6246 Field("bcc", "string", writable=False),
6247 Field("size", "integer", writable=False),
6248 Field("deleted", "boolean"),
6249 Field("draft", "boolean"),
6250 Field("flagged", "boolean"),
6251 Field("sender", "string", writable=False),
6252 Field("recent", "boolean", writable=False),
6253 Field("seen", "boolean"),
6254 Field("subject", "string", writable=False),
6255 Field("mime", "string", writable=False),
6256 Field("email", "string", writable=False, readable=False),
6257 Field("attachments", list, writable=False, readable=False),
6258 Field("encoding", writable=False)
6259 )
6260
6261
6262
6263 self.db[name].mailbox = \
6264 self.connection.mailbox_names[name]
6265
6266
6267 self.db[name].to.represent = self.db[name].cc.represent = \
6268 self.db[name].bcc.represent = self.db[name].sender.represent = \
6269 self.db[name].subject.represent = self.header_represent
6270
6271
6272 self.db.mailboxes = self.connection.mailbox_names
6273 return self.db.mailboxes
6274
6279
6280 - def _select(self, query, fields, attributes):
6284
6285 - def select(self, query, fields, attributes):
6286 """ Search and Fetch records and return web2py rows
6287 """
6288
6289 if use_common_filters(query):
6290 query = self.common_filter(query, [self.get_query_mailbox(query),])
6291
6292 import email
6293
6294
6295 tablename = None
6296 fetch_results = list()
6297
6298 if isinstance(query, Query):
6299 tablename = self.get_table(query)
6300 mailbox = self.connection.mailbox_names.get(tablename, None)
6301 if mailbox is None:
6302 raise ValueError("Mailbox name not found: %s" % mailbox)
6303 else:
6304
6305 result, selected = self.connection.select(mailbox, True)
6306 if result != "OK":
6307 raise Exception("IMAP error: %s" % selected)
6308 self.mailbox_size = int(selected[0])
6309 search_query = "(%s)" % str(query).strip()
6310 search_result = self.connection.uid("search", None, search_query)
6311
6312 if search_result[0] == "OK":
6313
6314
6315
6316
6317 limitby = attributes.get('limitby', None)
6318 messages_set = search_result[1][0].split()
6319
6320 messages_set.reverse()
6321 if limitby is not None:
6322
6323 messages_set = messages_set[int(limitby[0]):int(limitby[1])]
6324
6325
6326 if any([(field.name in ["content", "size",
6327 "attachments", "email"]) for
6328 field in fields]):
6329 imap_fields = "(RFC822 FLAGS)"
6330 else:
6331 imap_fields = "(RFC822.HEADER FLAGS)"
6332
6333 if len(messages_set) > 0:
6334
6335
6336
6337
6338 for uid in messages_set:
6339
6340 typ, data = self.connection.uid("fetch", uid, imap_fields)
6341 if typ == "OK":
6342 fr = {"message": int(data[0][0].split()[0]),
6343 "uid": long(uid),
6344 "email": email.message_from_string(data[0][1]),
6345 "raw_message": data[0][1]}
6346 fr["multipart"] = fr["email"].is_multipart()
6347
6348 fr["flags"] = self.driver.ParseFlags(data[1])
6349 fetch_results.append(fr)
6350 else:
6351
6352 raise Exception("IMAP error retrieving the body: %s" % data)
6353 else:
6354 raise Exception("IMAP search error: %s" % search_result[1])
6355 elif isinstance(query, (Expression, basestring)):
6356 raise NotImplementedError()
6357 else:
6358 raise TypeError("Unexpected query type")
6359
6360 imapqry_dict = {}
6361 imapfields_dict = {}
6362
6363 if len(fields) == 1 and isinstance(fields[0], SQLALL):
6364 allfields = True
6365 elif len(fields) == 0:
6366 allfields = True
6367 else:
6368 allfields = False
6369 if allfields:
6370 colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
6371 else:
6372 colnames = ["%s.%s" % (tablename, field.name) for field in fields]
6373
6374 for k in colnames:
6375 imapfields_dict[k] = k
6376
6377 imapqry_list = list()
6378 imapqry_array = list()
6379 for fr in fetch_results:
6380 attachments = []
6381 content = []
6382 size = 0
6383 n = int(fr["message"])
6384 item_dict = dict()
6385 message = fr["email"]
6386 uid = fr["uid"]
6387 charset = self.get_charset(message)
6388 flags = fr["flags"]
6389 raw_message = fr["raw_message"]
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400 if "%s.id" % tablename in colnames:
6401 item_dict["%s.id" % tablename] = n
6402 if "%s.created" % tablename in colnames:
6403 item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
6404 if "%s.uid" % tablename in colnames:
6405 item_dict["%s.uid" % tablename] = uid
6406 if "%s.sender" % tablename in colnames:
6407
6408
6409
6410 item_dict["%s.sender" % tablename] = message["From"]
6411 if "%s.to" % tablename in colnames:
6412 item_dict["%s.to" % tablename] = message["To"]
6413 if "%s.cc" % tablename in colnames:
6414 if "Cc" in message.keys():
6415 item_dict["%s.cc" % tablename] = message["Cc"]
6416 else:
6417 item_dict["%s.cc" % tablename] = ""
6418 if "%s.bcc" % tablename in colnames:
6419 if "Bcc" in message.keys():
6420 item_dict["%s.bcc" % tablename] = message["Bcc"]
6421 else:
6422 item_dict["%s.bcc" % tablename] = ""
6423 if "%s.deleted" % tablename in colnames:
6424 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
6425 if "%s.draft" % tablename in colnames:
6426 item_dict["%s.draft" % tablename] = "\\Draft" in flags
6427 if "%s.flagged" % tablename in colnames:
6428 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
6429 if "%s.recent" % tablename in colnames:
6430 item_dict["%s.recent" % tablename] = "\\Recent" in flags
6431 if "%s.seen" % tablename in colnames:
6432 item_dict["%s.seen" % tablename] = "\\Seen" in flags
6433 if "%s.subject" % tablename in colnames:
6434 item_dict["%s.subject" % tablename] = message["Subject"]
6435 if "%s.answered" % tablename in colnames:
6436 item_dict["%s.answered" % tablename] = "\\Answered" in flags
6437 if "%s.mime" % tablename in colnames:
6438 item_dict["%s.mime" % tablename] = message.get_content_type()
6439 if "%s.encoding" % tablename in colnames:
6440 item_dict["%s.encoding" % tablename] = charset
6441
6442
6443
6444
6445
6446
6447 if "%s.email" % tablename in colnames:
6448
6449 item_dict["%s.email" % tablename] = raw_message
6450
6451
6452
6453
6454
6455
6456 for part in message.walk():
6457 maintype = part.get_content_maintype()
6458 if ("%s.attachments" % tablename in colnames) or \
6459 ("%s.content" % tablename in colnames):
6460 payload = part.get_payload(decode=True)
6461 if payload:
6462 filename = part.get_filename()
6463 values = {"mime": part.get_content_type()}
6464 if ((filename or not "text" in maintype) and
6465 ("%s.attachments" % tablename in colnames)):
6466 values.update({"payload": payload,
6467 "filename": filename,
6468 "encoding": part.get_content_charset(),
6469 "disposition": part["Content-Disposition"]})
6470 attachments.append(values)
6471 elif (("text" in maintype) and
6472 ("%s.content" % tablename in colnames)):
6473 values.update({"text": self.encode_text(payload,
6474 self.get_charset(part))})
6475 content.append(values)
6476
6477 if "%s.size" % tablename in colnames:
6478 if part is not None:
6479 size += len(str(part))
6480 item_dict["%s.content" % tablename] = content
6481 item_dict["%s.attachments" % tablename] = attachments
6482 item_dict["%s.size" % tablename] = size
6483 imapqry_list.append(item_dict)
6484
6485
6486
6487 for item_dict in imapqry_list:
6488 imapqry_array_item = list()
6489 for fieldname in colnames:
6490 imapqry_array_item.append(item_dict[fieldname])
6491 imapqry_array.append(imapqry_array_item)
6492
6493
6494 colnames = colnames
6495 processor = attributes.get('processor',self.parse)
6496 return processor(imapqry_array, fields, colnames)
6497
6498 - def _insert(self, table, fields):
6499 def add_payload(message, obj):
6500 payload = Message()
6501 charset = obj.get("encoding", "utf-8")
6502 payload.set_type(obj.get("mime", None))
6503 payload.set_charset(charset)
6504 if "text" in obj:
6505 payload.set_payload(obj["text"])
6506 elif "payload" in obj:
6507 payload.set_payload(obj["payload"])
6508 if "filename" in obj and obj["filename"]:
6509 payload.add_header("Content-Disposition",
6510 "attachment", filename=obj["filename"])
6511 message.attach(payload)
6512
6513 mailbox = table.mailbox
6514 d = dict(((k.name, v) for k, v in fields))
6515 date_time = (d.get("created", datetime.datetime.now())).timetuple()
6516 if len(d) > 0:
6517 message = d.get("email", None)
6518 attachments = d.get("attachments", [])
6519 content = d.get("content", [])
6520 flags = " ".join(["\\%s" % flag.capitalize() for flag in
6521 ("answered", "deleted", "draft", "flagged",
6522 "recent", "seen") if d.get(flag, False)])
6523 if not message:
6524 from email.message import Message
6525 mime = d.get("mime", None)
6526 charset = d.get("encoding", None)
6527 message = Message()
6528 message["from"] = d.get("sender", "")
6529 message["subject"] = d.get("subject", "")
6530 if mime:
6531 message.set_type(mime)
6532 if charset:
6533 message.set_charset(charset)
6534 for item in ("to", "cc", "bcc"):
6535 value = d.get(item, "")
6536 if isinstance(value, basestring):
6537 message[item] = value
6538 else:
6539 message[item] = ";".join([i for i in
6540 value])
6541 if not message.is_multipart():
6542 if isinstance(content, basestring):
6543 message.set_payload(content)
6544 elif len(content) > 0:
6545 message.set_payload(content[0]["text"])
6546 else:
6547 [add_payload(message, c) for c in content]
6548 [add_payload(message, a) for a in attachments]
6549 message = message.as_string()
6550 return (mailbox, flags, date_time, message)
6551 else:
6552 raise NotImplementedError("IMAP empty insert is not implemented")
6553
6554 - def insert(self, table, fields):
6555 values = self._insert(table, fields)
6556 result, data = self.connection.append(*values)
6557 if result == "OK":
6558 uid = int(re.findall("\d+", str(data))[-1])
6559 return self.db(table.uid==uid).select(table.id).first().id
6560 else:
6561 raise Exception("IMAP message append failed: %s" % data)
6562
6563 - def _update(self, tablename, query, fields, commit=False):
6564
6565 commands = list()
6566 if use_common_filters(query):
6567 query = self.common_filter(query, [tablename,])
6568 mark = []
6569 unmark = []
6570 if query:
6571 for item in fields:
6572 field = item[0]
6573 name = field.name
6574 value = item[1]
6575 if self.is_flag(name):
6576 flag = self.search_fields[name]
6577 if (value is not None) and (flag != "\\Recent"):
6578 if value:
6579 mark.append(flag)
6580 else:
6581 unmark.append(flag)
6582 result, data = self.connection.select(
6583 self.connection.mailbox_names[tablename])
6584 string_query = "(%s)" % query
6585 result, data = self.connection.search(None, string_query)
6586 store_list = [item.strip() for item in data[0].split()
6587 if item.strip().isdigit()]
6588
6589 for number in store_list:
6590 result = None
6591 if len(mark) > 0:
6592 commands.append((number, "+FLAGS", "(%s)" % " ".join(mark)))
6593 if len(unmark) > 0:
6594 commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark)))
6595 return commands
6596
6597 - def update(self, tablename, query, fields):
6598 rowcount = 0
6599 commands = self._update(tablename, query, fields)
6600 for command in commands:
6601 result, data = self.connection.store(*command)
6602 if result == "OK":
6603 rowcount += 1
6604 else:
6605 raise Exception("IMAP storing error: %s" % data)
6606 return rowcount
6607
6608 - def _count(self, query, distinct=None):
6609 raise NotImplementedError()
6610
6611 - def count(self,query,distinct=None):
6623
6624 - def delete(self, tablename, query):
6625 counter = 0
6626 if query:
6627 if use_common_filters(query):
6628 query = self.common_filter(query, [tablename,])
6629 result, data = self.connection.select(self.connection.mailbox_names[tablename])
6630 string_query = "(%s)" % query
6631 result, data = self.connection.search(None, string_query)
6632 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
6633 for number in store_list:
6634 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
6635 if result == "OK":
6636 counter += 1
6637 else:
6638 raise Exception("IMAP store error: %s" % data)
6639 if counter > 0:
6640 result, data = self.connection.expunge()
6641 return counter
6642
6643 - def BELONGS(self, first, second):
6644 result = None
6645 name = self.search_fields[first.name]
6646 if name == "MESSAGE":
6647 values = [str(val) for val in second if str(val).isdigit()]
6648 result = "%s" % ",".join(values).strip()
6649
6650 elif name == "UID":
6651 values = [str(val) for val in second if str(val).isdigit()]
6652 result = "UID %s" % ",".join(values).strip()
6653
6654 else:
6655 raise Exception("Operation not supported")
6656
6657 return result
6658
6659 - def CONTAINS(self, first, second, case_sensitive=False):
6660
6661 result = None
6662 name = self.search_fields[first.name]
6663
6664 if name in ("FROM", "TO", "SUBJECT", "TEXT"):
6665 result = "%s \"%s\"" % (name, self.expand(second))
6666 else:
6667 if first.name in ("cc", "bcc"):
6668 result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
6669 elif first.name == "mime":
6670 result = "HEADER Content-Type \"%s\"" % self.expand(second)
6671 else:
6672 raise Exception("Operation not supported")
6673 return result
6674
6675 - def GT(self, first, second):
6676 result = None
6677 name = self.search_fields[first.name]
6678 if name == "MESSAGE":
6679 last_message = self.get_last_message(first.tablename)
6680 result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
6681 elif name == "UID":
6682
6683
6684
6685 try:
6686 pedestal, threshold = self.get_uid_bounds(first.tablename)
6687 except TypeError:
6688 e = sys.exc_info()[1]
6689 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6690 return ""
6691 try:
6692 lower_limit = int(self.expand(second)) + 1
6693 except (ValueError, TypeError):
6694 e = sys.exc_info()[1]
6695 raise Exception("Operation not supported (non integer UID)")
6696 result = "UID %s:%s" % (lower_limit, threshold)
6697 elif name == "DATE":
6698 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
6699 elif name == "SIZE":
6700 result = "LARGER %s" % self.expand(second)
6701 else:
6702 raise Exception("Operation not supported")
6703 return result
6704
6705 - def GE(self, first, second):
6706 result = None
6707 name = self.search_fields[first.name]
6708 if name == "MESSAGE":
6709 last_message = self.get_last_message(first.tablename)
6710 result = "%s:%s" % (self.expand(second), last_message)
6711 elif name == "UID":
6712
6713
6714
6715 try:
6716 pedestal, threshold = self.get_uid_bounds(first.tablename)
6717 except TypeError:
6718 e = sys.exc_info()[1]
6719 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6720 return ""
6721 lower_limit = self.expand(second)
6722 result = "UID %s:%s" % (lower_limit, threshold)
6723 elif name == "DATE":
6724 result = "SINCE %s" % self.convert_date(second)
6725 else:
6726 raise Exception("Operation not supported")
6727 return result
6728
6729 - def LT(self, first, second):
6730 result = None
6731 name = self.search_fields[first.name]
6732 if name == "MESSAGE":
6733 result = "%s:%s" % (1, int(self.expand(second)) - 1)
6734 elif name == "UID":
6735 try:
6736 pedestal, threshold = self.get_uid_bounds(first.tablename)
6737 except TypeError:
6738 e = sys.exc_info()[1]
6739 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6740 return ""
6741 try:
6742 upper_limit = int(self.expand(second)) - 1
6743 except (ValueError, TypeError):
6744 e = sys.exc_info()[1]
6745 raise Exception("Operation not supported (non integer UID)")
6746 result = "UID %s:%s" % (pedestal, upper_limit)
6747 elif name == "DATE":
6748 result = "BEFORE %s" % self.convert_date(second)
6749 elif name == "SIZE":
6750 result = "SMALLER %s" % self.expand(second)
6751 else:
6752 raise Exception("Operation not supported")
6753 return result
6754
6755 - def LE(self, first, second):
6756 result = None
6757 name = self.search_fields[first.name]
6758 if name == "MESSAGE":
6759 result = "%s:%s" % (1, self.expand(second))
6760 elif name == "UID":
6761 try:
6762 pedestal, threshold = self.get_uid_bounds(first.tablename)
6763 except TypeError:
6764 e = sys.exc_info()[1]
6765 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6766 return ""
6767 upper_limit = int(self.expand(second))
6768 result = "UID %s:%s" % (pedestal, upper_limit)
6769 elif name == "DATE":
6770 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
6771 else:
6772 raise Exception("Operation not supported")
6773 return result
6774
6775 - def NE(self, first, second=None):
6776 if (second is None) and isinstance(first, Field):
6777
6778 if first.type == "id":
6779 return self.GE(first, 1)
6780 result = self.NOT(self.EQ(first, second))
6781 result = result.replace("NOT NOT", "").strip()
6782 return result
6783
6784 - def EQ(self,first,second):
6785 name = self.search_fields[first.name]
6786 result = None
6787 if name is not None:
6788 if name == "MESSAGE":
6789
6790 result = "%s" % self.expand(second)
6791 elif name == "UID":
6792 result = "UID %s" % self.expand(second)
6793 elif name == "DATE":
6794 result = "ON %s" % self.convert_date(second)
6795
6796 elif name in self.flags.values():
6797 if second:
6798 result = "%s" % (name.upper()[1:])
6799 else:
6800 result = "NOT %s" % (name.upper()[1:])
6801 else:
6802 raise Exception("Operation not supported")
6803 else:
6804 raise Exception("Operation not supported")
6805 return result
6806
6807 - def AND(self, first, second):
6810
6811 - def OR(self, first, second):
6814
6815 - def NOT(self, first):
6816 result = "NOT %s" % self.expand(first)
6817 return result
6818
6819
6820
6821
6822
6823 ADAPTERS = {
6824 'sqlite': SQLiteAdapter,
6825 'spatialite': SpatiaLiteAdapter,
6826 'sqlite:memory': SQLiteAdapter,
6827 'spatialite:memory': SpatiaLiteAdapter,
6828 'mysql': MySQLAdapter,
6829 'postgres': PostgreSQLAdapter,
6830 'postgres:psycopg2': PostgreSQLAdapter,
6831 'postgres:pg8000': PostgreSQLAdapter,
6832 'postgres2:psycopg2': NewPostgreSQLAdapter,
6833 'postgres2:pg8000': NewPostgreSQLAdapter,
6834 'oracle': OracleAdapter,
6835 'mssql': MSSQLAdapter,
6836 'mssql2': MSSQL2Adapter,
6837 'mssql3': MSSQL3Adapter,
6838 'vertica': VerticaAdapter,
6839 'sybase': SybaseAdapter,
6840 'db2': DB2Adapter,
6841 'teradata': TeradataAdapter,
6842 'informix': InformixAdapter,
6843 'informix-se': InformixSEAdapter,
6844 'firebird': FireBirdAdapter,
6845 'firebird_embedded': FireBirdAdapter,
6846 'ingres': IngresAdapter,
6847 'ingresu': IngresUnicodeAdapter,
6848 'sapdb': SAPDBAdapter,
6849 'cubrid': CubridAdapter,
6850 'jdbc:sqlite': JDBCSQLiteAdapter,
6851 'jdbc:sqlite:memory': JDBCSQLiteAdapter,
6852 'jdbc:postgres': JDBCPostgreSQLAdapter,
6853 'gae': GoogleDatastoreAdapter,
6854 'google:datastore': GoogleDatastoreAdapter,
6855 'google:sql': GoogleSQLAdapter,
6856 'couchdb': CouchDBAdapter,
6857 'mongodb': MongoDBAdapter,
6858 'imap': IMAPAdapter
6859 }
6862 """
6863 Field type validation, using web2py's validators mechanism.
6864
6865 makes sure the content of a field is in line with the declared
6866 fieldtype
6867 """
6868 db = field.db
6869 try:
6870 from gluon import validators
6871 except ImportError:
6872 return []
6873 field_type, field_length = field.type, field.length
6874 if isinstance(field_type, SQLCustomType):
6875 if hasattr(field_type, 'validator'):
6876 return field_type.validator
6877 else:
6878 field_type = field_type.type
6879 elif not isinstance(field_type,str):
6880 return []
6881 requires=[]
6882 def ff(r,id):
6883 row=r(id)
6884 if not row:
6885 return id
6886 elif hasattr(r, '_format') and isinstance(r._format,str):
6887 return r._format % row
6888 elif hasattr(r, '_format') and callable(r._format):
6889 return r._format(row)
6890 else:
6891 return id
6892 if field_type in (('string', 'text', 'password')):
6893 requires.append(validators.IS_LENGTH(field_length))
6894 elif field_type == 'json':
6895 requires.append(validators.IS_EMPTY_OR(validators.IS_JSON(native_json=field.db._adapter.native_json)))
6896 elif field_type == 'double' or field_type == 'float':
6897 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
6898 elif field_type in ('integer','bigint'):
6899 requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100))
6900 elif field_type.startswith('decimal'):
6901 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
6902 elif field_type == 'date':
6903 requires.append(validators.IS_DATE())
6904 elif field_type == 'time':
6905 requires.append(validators.IS_TIME())
6906 elif field_type == 'datetime':
6907 requires.append(validators.IS_DATETIME())
6908 elif db and field_type.startswith('reference') and \
6909 field_type.find('.') < 0 and \
6910 field_type[10:] in db.tables:
6911 referenced = db[field_type[10:]]
6912 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
6913 field.represent = field.represent or repr_ref
6914 if hasattr(referenced, '_format') and referenced._format:
6915 requires = validators.IS_IN_DB(db,referenced._id,
6916 referenced._format)
6917 if field.unique:
6918 requires._and = validators.IS_NOT_IN_DB(db,field)
6919 if field.tablename == field_type[10:]:
6920 return validators.IS_EMPTY_OR(requires)
6921 return requires
6922 elif db and field_type.startswith('list:reference') and \
6923 field_type.find('.') < 0 and \
6924 field_type[15:] in db.tables:
6925 referenced = db[field_type[15:]]
6926 def list_ref_repr(ids, row=None, r=referenced, f=ff):
6927 if not ids:
6928 return None
6929 refs = None
6930 db, id = r._db, r._id
6931 if isinstance(db._adapter, GoogleDatastoreAdapter):
6932 def count(values): return db(id.belongs(values)).select(id)
6933 rx = range(0, len(ids), 30)
6934 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
6935 else:
6936 refs = db(id.belongs(ids)).select(id)
6937 return (refs and ', '.join(f(r,x.id) for x in refs) or '')
6938 field.represent = field.represent or list_ref_repr
6939 if hasattr(referenced, '_format') and referenced._format:
6940 requires = validators.IS_IN_DB(db,referenced._id,
6941 referenced._format,multiple=True)
6942 else:
6943 requires = validators.IS_IN_DB(db,referenced._id,
6944 multiple=True)
6945 if field.unique:
6946 requires._and = validators.IS_NOT_IN_DB(db,field)
6947 if not field.notnull:
6948 requires = validators.IS_EMPTY_OR(requires)
6949 return requires
6950 elif field_type.startswith('list:'):
6951 def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
6952 field.represent = field.represent or repr_list
6953 if field.unique:
6954 requires.insert(0,validators.IS_NOT_IN_DB(db,field))
6955 sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
6956 if field.notnull and not field_type[:2] in sff:
6957 requires.insert(0, validators.IS_NOT_EMPTY())
6958 elif not field.notnull and field_type[:2] in sff and requires:
6959 requires[-1] = validators.IS_EMPTY_OR(requires[-1])
6960 return requires
6961
6964 return str(item).replace('|', '||')
6965
6968
6970 if not hasattr(value,'split') and hasattr(value,'read'):
6971 value = value.read()
6972 return [long(x) for x in value.split('|') if x.strip()]
6973
6977
6978
6979 -class Row(object):
6980
6981 """
6982 a dictionary that lets you do d['a'] as well as d.a
6983 this is only used to store a Row
6984 """
6985
6986 __init__ = lambda self,*args,**kwargs: self.__dict__.update(*args,**kwargs)
6987
6989 key=str(k)
6990 _extra = self.__dict__.get('_extra', None)
6991 if _extra is not None:
6992 v = _extra.get(key, DEFAULT)
6993 if v != DEFAULT:
6994 return v
6995 m = REGEX_TABLE_DOT_FIELD.match(key)
6996 if m:
6997 try:
6998 return ogetattr(self, m.group(1))[m.group(2)]
6999 except (KeyError,AttributeError,TypeError):
7000 key = m.group(2)
7001 try:
7002 return ogetattr(self, key)
7003 except (KeyError,AttributeError,TypeError), ae:
7004 try:
7005 self[key] = ogetattr(self,'__get_lazy_reference__')(key)
7006 return self[key]
7007 except:
7008 raise ae
7009
7010 __setitem__ = lambda self, key, value: setattr(self, str(key), value)
7011
7012 __delitem__ = object.__delattr__
7013
7014 __copy__ = lambda self: Row(self)
7015
7016 __call__ = __getitem__
7017
7018
7019 - def get(self, key, default=None):
7020 try:
7021 return self.__getitem__(key)
7022 except(KeyError, AttributeError, TypeError):
7023 return self.__dict__.get(key,default)
7024
7025 has_key = __contains__ = lambda self, key: key in self.__dict__
7026
7027 __nonzero__ = lambda self: len(self.__dict__)>0
7028
7029 update = lambda self, *args, **kwargs: self.__dict__.update(*args, **kwargs)
7030
7031 keys = lambda self: self.__dict__.keys()
7032
7033 items = lambda self: self.__dict__.items()
7034
7035 values = lambda self: self.__dict__.values()
7036
7037 __iter__ = lambda self: self.__dict__.__iter__()
7038
7039 iteritems = lambda self: self.__dict__.iteritems()
7040
7041 __str__ = __repr__ = lambda self: '<Row %s>' % self.as_dict()
7042
7043 __int__ = lambda self: object.__getattribute__(self,'id')
7044
7045 __long__ = lambda self: long(object.__getattribute__(self,'id'))
7046
7047 __getattr__ = __getitem__
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7059 try:
7060 return self.as_dict() == other.as_dict()
7061 except AttributeError:
7062 return False
7063
7065 return not (self == other)
7066
7068 return Row(dict(self))
7069
7070 - def as_dict(self, datetime_to_str=False, custom_types=None):
7071 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict]
7072 if isinstance(custom_types,(list,tuple,set)):
7073 SERIALIZABLE_TYPES += custom_types
7074 elif custom_types:
7075 SERIALIZABLE_TYPES.append(custom_types)
7076 d = dict(self)
7077 for k in copy.copy(d.keys()):
7078 v=d[k]
7079 if d[k] is None:
7080 continue
7081 elif isinstance(v,Row):
7082 d[k]=v.as_dict()
7083 elif isinstance(v,Reference):
7084 d[k]=long(v)
7085 elif isinstance(v,decimal.Decimal):
7086 d[k]=float(v)
7087 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
7088 if datetime_to_str:
7089 d[k] = v.isoformat().replace('T',' ')[:19]
7090 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)):
7091 del d[k]
7092 return d
7093
7094 - def as_xml(self, row_name="row", colnames=None, indent=' '):
7095 def f(row,field,indent=' '):
7096 if isinstance(row,Row):
7097 spc = indent+' \n'
7098 items = [f(row[x],x,indent+' ') for x in row]
7099 return '%s<%s>\n%s\n%s</%s>' % (
7100 indent,
7101 field,
7102 spc.join(item for item in items if item),
7103 indent,
7104 field)
7105 elif not callable(row):
7106 if REGEX_ALPHANUMERIC.match(field):
7107 return '%s<%s>%s</%s>' % (indent,field,row,field)
7108 else:
7109 return '%s<extra name="%s">%s</extra>' % \
7110 (indent,field,row)
7111 else:
7112 return None
7113 return f(self, row_name, indent=indent)
7114
7115 - def as_json(self, mode="object", default=None, colnames=None,
7116 serialize=True, **kwargs):
7117 """
7118 serializes the row to a JSON object
7119 kwargs are passed to .as_dict method
7120 only "object" mode supported
7121
7122 serialize = False used by Rows.as_json
7123 TODO: return array mode with query column order
7124
7125 mode and colnames are not implemented
7126 """
7127
7128 item = self.as_dict(**kwargs)
7129 if serialize:
7130 if have_serializers:
7131 return serializers.json(item,
7132 default=default or
7133 serializers.custom_json)
7134 elif simplejson:
7135 return simplejson.dumps(item)
7136 else:
7137 raise RuntimeError("missing simplejson")
7138 else:
7139 return item
7140
7150
7152 if not isinstance(fields,(list,tuple)):
7153 fields = [fields]
7154 new_fields = []
7155 for field in fields:
7156 if isinstance(field,Field):
7157 new_fields.append(field)
7158 elif isinstance(field,Table):
7159 for ofield in field:
7160 new_fields.append(ofield)
7161 else:
7162 raise RuntimeError("fields must be a list of fields")
7163 fields = new_fields
7164 field_map = {}
7165 for field in fields:
7166 n = field.name.lower()
7167 if not n in field_map:
7168 field_map[n] = field
7169 n = str(field).lower()
7170 if not n in field_map:
7171 field_map[n] = field
7172 constants = {}
7173 i = 0
7174 while True:
7175 m = REGEX_CONST_STRING.search(text)
7176 if not m: break
7177 text = text[:m.start()]+('#%i' % i)+text[m.end():]
7178 constants[str(i)] = m.group()[1:-1]
7179 i+=1
7180 text = re.sub('\s+',' ',text).lower()
7181 for a,b in [('&','and'),
7182 ('|','or'),
7183 ('~','not'),
7184 ('==','='),
7185 ('<','<'),
7186 ('>','>'),
7187 ('<=','<='),
7188 ('>=','>='),
7189 ('<>','!='),
7190 ('=<','<='),
7191 ('=>','>='),
7192 ('=','='),
7193 (' less or equal than ','<='),
7194 (' greater or equal than ','>='),
7195 (' equal or less than ','<='),
7196 (' equal or greater than ','>='),
7197 (' less or equal ','<='),
7198 (' greater or equal ','>='),
7199 (' equal or less ','<='),
7200 (' equal or greater ','>='),
7201 (' not equal to ','!='),
7202 (' not equal ','!='),
7203 (' equal to ','='),
7204 (' equal ','='),
7205 (' equals ','='),
7206 (' less than ','<'),
7207 (' greater than ','>'),
7208 (' starts with ','startswith'),
7209 (' ends with ','endswith'),
7210 (' not in ' , 'notbelongs'),
7211 (' in ' , 'belongs'),
7212 (' is ','=')]:
7213 if a[0]==' ':
7214 text = text.replace(' is'+a,' %s ' % b)
7215 text = text.replace(a,' %s ' % b)
7216 text = re.sub('\s+',' ',text).lower()
7217 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
7218 query = field = neg = op = logic = None
7219 for item in text.split():
7220 if field is None:
7221 if item == 'not':
7222 neg = True
7223 elif not neg and not logic and item in ('and','or'):
7224 logic = item
7225 elif item in field_map:
7226 field = field_map[item]
7227 else:
7228 raise RuntimeError("Invalid syntax")
7229 elif not field is None and op is None:
7230 op = item
7231 elif not op is None:
7232 if item.startswith('#'):
7233 if not item[1:] in constants:
7234 raise RuntimeError("Invalid syntax")
7235 value = constants[item[1:]]
7236 else:
7237 value = item
7238 if field.type in ('text', 'string', 'json'):
7239 if op == '=': op = 'like'
7240 if op == '=': new_query = field==value
7241 elif op == '<': new_query = field<value
7242 elif op == '>': new_query = field>value
7243 elif op == '<=': new_query = field<=value
7244 elif op == '>=': new_query = field>=value
7245 elif op == '!=': new_query = field!=value
7246 elif op == 'belongs': new_query = field.belongs(value.split(','))
7247 elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
7248 elif field.type in ('text', 'string', 'json'):
7249 if op == 'contains': new_query = field.contains(value)
7250 elif op == 'like': new_query = field.like(value)
7251 elif op == 'startswith': new_query = field.startswith(value)
7252 elif op == 'endswith': new_query = field.endswith(value)
7253 else: raise RuntimeError("Invalid operation")
7254 elif field._db._adapter.dbengine=='google:datastore' and \
7255 field.type in ('list:integer', 'list:string', 'list:reference'):
7256 if op == 'contains': new_query = field.contains(value)
7257 else: raise RuntimeError("Invalid operation")
7258 else: raise RuntimeError("Invalid operation")
7259 if neg: new_query = ~new_query
7260 if query is None:
7261 query = new_query
7262 elif logic == 'and':
7263 query &= new_query
7264 elif logic == 'or':
7265 query |= new_query
7266 field = op = neg = logic = None
7267 return query
7268
7270
7271 """
7272 an instance of this class represents a database connection
7273
7274 Example::
7275
7276 db = DAL('sqlite://test.db')
7277
7278 or
7279
7280 db = DAL(**{"uri": ..., "tables": [...]...}) # experimental
7281
7282 db.define_table('tablename', Field('fieldname1'),
7283 Field('fieldname2'))
7284 """
7285
7286 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
7313
7314 @staticmethod
7316 """
7317 # ## this allows gluon to set a folder for this thread
7318 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
7319 """
7320 BaseAdapter.set_folder(folder)
7321
7322 @staticmethod
7324 """
7325 Returns a dictionary with uri as key with timings and defined tables
7326 {'sqlite://storage.sqlite': {
7327 'dbstats': [(select auth_user.email from auth_user, 0.02009)],
7328 'dbtables': {
7329 'defined': ['auth_cas', 'auth_event', 'auth_group',
7330 'auth_membership', 'auth_permission', 'auth_user'],
7331 'lazy': '[]'
7332 }
7333 }
7334 }
7335 """
7336 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
7337 infos = {}
7338 for db_uid, db_group in dbs:
7339 for db in db_group:
7340 if not db._uri:
7341 continue
7342 k = hide_password(db._uri)
7343 infos[k] = dict(dbstats = [(row[0], row[1]) for row in db._timings],
7344 dbtables = {'defined':
7345 sorted(list(set(db.tables) -
7346 set(db._LAZY_TABLES.keys()))),
7347 'lazy': sorted(db._LAZY_TABLES.keys())}
7348 )
7349 return infos
7350
7351 @staticmethod
7364
7365 @staticmethod
7387
7388 - def __init__(self, uri=DEFAULT_URI,
7389 pool_size=0, folder=None,
7390 db_codec='UTF-8', check_reserved=None,
7391 migrate=True, fake_migrate=False,
7392 migrate_enabled=True, fake_migrate_all=False,
7393 decode_credentials=False, driver_args=None,
7394 adapter_args=None, attempts=5, auto_import=False,
7395 bigint_id=False, debug=False, lazy_tables=False,
7396 db_uid=None, do_connect=True,
7397 after_connection=None, tables=None):
7398 """
7399 Creates a new Database Abstraction Layer instance.
7400
7401 Keyword arguments:
7402
7403 :uri: string that contains information for connecting to a database.
7404 (default: 'sqlite://dummy.db')
7405
7406 experimental: you can specify a dictionary as uri
7407 parameter i.e. with
7408 db = DAL({"uri": "sqlite://storage.sqlite",
7409 "tables": {...}, ...})
7410
7411 for an example of dict input you can check the output
7412 of the scaffolding db model with
7413
7414 db.as_dict()
7415
7416 Note that for compatibility with Python older than
7417 version 2.6.5 you should cast your dict input keys
7418 to str due to a syntax limitation on kwarg names.
7419 for proper DAL dictionary input you can use one of:
7420
7421 obj = serializers.cast_keys(dict, [encoding="utf-8"])
7422
7423 or else (for parsing json input)
7424
7425 obj = serializers.loads_json(data, unicode_keys=False)
7426
7427 :pool_size: How many open connections to make to the database object.
7428 :folder: where .table files will be created.
7429 automatically set within web2py
7430 use an explicit path when using DAL outside web2py
7431 :db_codec: string encoding of the database (default: 'UTF-8')
7432 :check_reserved: list of adapters to check tablenames and column names
7433 against sql/nosql reserved keywords. (Default None)
7434
7435 * 'common' List of sql keywords that are common to all database types
7436 such as "SELECT, INSERT". (recommended)
7437 * 'all' Checks against all known SQL keywords. (not recommended)
7438 <adaptername> Checks against the specific adapters list of keywords
7439 (recommended)
7440 * '<adaptername>_nonreserved' Checks against the specific adapters
7441 list of nonreserved keywords. (if available)
7442 :migrate (defaults to True) sets default migrate behavior for all tables
7443 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
7444 :migrate_enabled (defaults to True). If set to False disables ALL migrations
7445 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
7446 :attempts (defaults to 5). Number of times to attempt connecting
7447 :auto_import (defaults to False). If set, import automatically table definitions from the
7448 databases folder
7449 :bigint_id (defaults to False): If set, turn on bigint instead of int for id fields
7450 :lazy_tables (defaults to False): delay table definition until table access
7451 :after_connection (defaults to None): a callable that will be execute after the connection
7452 """
7453 if uri == '<zombie>' and db_uid is not None: return
7454 if not decode_credentials:
7455 credential_decoder = lambda cred: cred
7456 else:
7457 credential_decoder = lambda cred: urllib.unquote(cred)
7458 self._folder = folder
7459 if folder:
7460 self.set_folder(folder)
7461 self._uri = uri
7462 self._pool_size = pool_size
7463 self._db_codec = db_codec
7464 self._lastsql = ''
7465 self._timings = []
7466 self._pending_references = {}
7467 self._request_tenant = 'request_tenant'
7468 self._common_fields = []
7469 self._referee_name = '%(table)s'
7470 self._bigint_id = bigint_id
7471 self._debug = debug
7472 self._migrated = []
7473 self._LAZY_TABLES = {}
7474 self._lazy_tables = lazy_tables
7475 self._tables = SQLCallableList()
7476 self._driver_args = driver_args
7477 self._adapter_args = adapter_args
7478 self._check_reserved = check_reserved
7479 self._decode_credentials = decode_credentials
7480 self._attempts = attempts
7481 self._do_connect = do_connect
7482
7483 if not str(attempts).isdigit() or attempts < 0:
7484 attempts = 5
7485 if uri:
7486 uris = isinstance(uri,(list,tuple)) and uri or [uri]
7487 error = ''
7488 connected = False
7489 for k in range(attempts):
7490 for uri in uris:
7491 try:
7492 if is_jdbc and not uri.startswith('jdbc:'):
7493 uri = 'jdbc:'+uri
7494 self._dbname = REGEX_DBNAME.match(uri).group()
7495 if not self._dbname in ADAPTERS:
7496 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname)
7497
7498
7499 kwargs = dict(db=self,uri=uri,
7500 pool_size=pool_size,
7501 folder=folder,
7502 db_codec=db_codec,
7503 credential_decoder=credential_decoder,
7504 driver_args=driver_args or {},
7505 adapter_args=adapter_args or {},
7506 do_connect=do_connect,
7507 after_connection=after_connection)
7508 self._adapter = ADAPTERS[self._dbname](**kwargs)
7509 types = ADAPTERS[self._dbname].types
7510
7511 self._adapter.types = copy.copy(types)
7512 self._adapter.build_parsemap()
7513 if bigint_id:
7514 if 'big-id' in types and 'reference' in types:
7515 self._adapter.types['id'] = types['big-id']
7516 self._adapter.types['reference'] = types['big-reference']
7517 connected = True
7518 break
7519 except SyntaxError:
7520 raise
7521 except Exception:
7522 tb = traceback.format_exc()
7523 sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
7524 if connected:
7525 break
7526 else:
7527 time.sleep(1)
7528 if not connected:
7529 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb))
7530 else:
7531 self._adapter = BaseAdapter(db=self,pool_size=0,
7532 uri='None',folder=folder,
7533 db_codec=db_codec, after_connection=after_connection)
7534 migrate = fake_migrate = False
7535 adapter = self._adapter
7536 self._uri_hash = hashlib_md5(adapter.uri).hexdigest()
7537 self.check_reserved = check_reserved
7538 if self.check_reserved:
7539 from reserved_sql_keywords import ADAPTERS as RSK
7540 self.RSK = RSK
7541 self._migrate = migrate
7542 self._fake_migrate = fake_migrate
7543 self._migrate_enabled = migrate_enabled
7544 self._fake_migrate_all = fake_migrate_all
7545 if auto_import or tables:
7546 self.import_table_definitions(adapter.folder,
7547 tables=tables)
7548
7549 @property
7552
7555 pattern = pjoin(path,self._uri_hash+'_*.table')
7556 if tables:
7557 for table in tables:
7558 self.define_table(**table)
7559 else:
7560 for filename in glob.glob(pattern):
7561 tfile = self._adapter.file_open(filename, 'r')
7562 try:
7563 sql_fields = pickle.load(tfile)
7564 name = filename[len(pattern)-7:-6]
7565 mf = [(value['sortable'],
7566 Field(key,
7567 type=value['type'],
7568 length=value.get('length',None),
7569 notnull=value.get('notnull',False),
7570 unique=value.get('unique',False))) \
7571 for key, value in sql_fields.iteritems()]
7572 mf.sort(lambda a,b: cmp(a[0],b[0]))
7573 self.define_table(name,*[item[1] for item in mf],
7574 **dict(migrate=migrate,
7575 fake_migrate=fake_migrate))
7576 finally:
7577 self._adapter.file_close(tfile)
7578
7580 """
7581 Validates ``name`` against SQL keywords
7582 Uses self.check_reserve which is a list of
7583 operators to use.
7584 self.check_reserved
7585 ['common', 'postgres', 'mysql']
7586 self.check_reserved
7587 ['all']
7588 """
7589 for backend in self.check_reserved:
7590 if name.upper() in self.RSK[backend]:
7591 raise SyntaxError(
7592 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper()))
7593
7594 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
7595 """
7596 EXAMPLE:
7597
7598 db.define_table('person',Field('name'),Field('info'))
7599 db.define_table('pet',Field('ownedby',db.person),Field('name'),Field('info'))
7600
7601 @request.restful()
7602 def index():
7603 def GET(*args,**vars):
7604 patterns = [
7605 "/friends[person]",
7606 "/{person.name}/:field",
7607 "/{person.name}/pets[pet.ownedby]",
7608 "/{person.name}/pets[pet.ownedby]/{pet.name}",
7609 "/{person.name}/pets[pet.ownedby]/{pet.name}/:field",
7610 ("/dogs[pet]", db.pet.info=='dog'),
7611 ("/dogs[pet]/{pet.name.startswith}", db.pet.info=='dog'),
7612 ]
7613 parser = db.parse_as_rest(patterns,args,vars)
7614 if parser.status == 200:
7615 return dict(content=parser.response)
7616 else:
7617 raise HTTP(parser.status,parser.error)
7618
7619 def POST(table_name,**vars):
7620 if table_name == 'person':
7621 return db.person.validate_and_insert(**vars)
7622 elif table_name == 'pet':
7623 return db.pet.validate_and_insert(**vars)
7624 else:
7625 raise HTTP(400)
7626 return locals()
7627 """
7628
7629 db = self
7630 re1 = REGEX_SEARCH_PATTERN
7631 re2 = REGEX_SQUARE_BRACKETS
7632
7633 def auto_table(table,base='',depth=0):
7634 patterns = []
7635 for field in db[table].fields:
7636 if base:
7637 tag = '%s/%s' % (base,field.replace('_','-'))
7638 else:
7639 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
7640 f = db[table][field]
7641 if not f.readable: continue
7642 if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
7643 tag += '/{%s.%s}' % (table,field)
7644 patterns.append(tag)
7645 patterns.append(tag+'/:field')
7646 elif f.type.startswith('boolean'):
7647 tag += '/{%s.%s}' % (table,field)
7648 patterns.append(tag)
7649 patterns.append(tag+'/:field')
7650 elif f.type in ('float','double','integer','bigint'):
7651 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
7652 patterns.append(tag)
7653 patterns.append(tag+'/:field')
7654 elif f.type.startswith('list:'):
7655 tag += '/{%s.%s.contains}' % (table,field)
7656 patterns.append(tag)
7657 patterns.append(tag+'/:field')
7658 elif f.type in ('date','datetime'):
7659 tag+= '/{%s.%s.year}' % (table,field)
7660 patterns.append(tag)
7661 patterns.append(tag+'/:field')
7662 tag+='/{%s.%s.month}' % (table,field)
7663 patterns.append(tag)
7664 patterns.append(tag+'/:field')
7665 tag+='/{%s.%s.day}' % (table,field)
7666 patterns.append(tag)
7667 patterns.append(tag+'/:field')
7668 if f.type in ('datetime','time'):
7669 tag+= '/{%s.%s.hour}' % (table,field)
7670 patterns.append(tag)
7671 patterns.append(tag+'/:field')
7672 tag+='/{%s.%s.minute}' % (table,field)
7673 patterns.append(tag)
7674 patterns.append(tag+'/:field')
7675 tag+='/{%s.%s.second}' % (table,field)
7676 patterns.append(tag)
7677 patterns.append(tag+'/:field')
7678 if depth>0:
7679 for f in db[table]._referenced_by:
7680 tag+='/%s[%s.%s]' % (table,f.tablename,f.name)
7681 patterns.append(tag)
7682 patterns += auto_table(table,base=tag,depth=depth-1)
7683 return patterns
7684
7685 if patterns == 'auto':
7686 patterns=[]
7687 for table in db.tables:
7688 if not table.startswith('auth_'):
7689 patterns.append('/%s[%s]' % (table,table))
7690 patterns += auto_table(table,base='',depth=1)
7691 else:
7692 i = 0
7693 while i<len(patterns):
7694 pattern = patterns[i]
7695 if not isinstance(pattern,str):
7696 pattern = pattern[0]
7697 tokens = pattern.split('/')
7698 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
7699 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
7700 '/'.join(tokens[:-1]))
7701 patterns = patterns[:i]+new_patterns+patterns[i+1:]
7702 i += len(new_patterns)
7703 else:
7704 i += 1
7705 if '/'.join(args) == 'patterns':
7706 return Row({'status':200,'pattern':'list',
7707 'error':None,'response':patterns})
7708 for pattern in patterns:
7709 basequery, exposedfields = None, []
7710 if isinstance(pattern,tuple):
7711 if len(pattern)==2:
7712 pattern, basequery = pattern
7713 elif len(pattern)>2:
7714 pattern, basequery, exposedfields = pattern[0:3]
7715 otable=table=None
7716 if not isinstance(queries,dict):
7717 dbset=db(queries)
7718 if basequery is not None:
7719 dbset = dbset(basequery)
7720 i=0
7721 tags = pattern[1:].split('/')
7722 if len(tags)!=len(args):
7723 continue
7724 for tag in tags:
7725 if re1.match(tag):
7726
7727 tokens = tag[1:-1].split('.')
7728 table, field = tokens[0], tokens[1]
7729 if not otable or table == otable:
7730 if len(tokens)==2 or tokens[2]=='eq':
7731 query = db[table][field]==args[i]
7732 elif tokens[2]=='ne':
7733 query = db[table][field]!=args[i]
7734 elif tokens[2]=='lt':
7735 query = db[table][field]<args[i]
7736 elif tokens[2]=='gt':
7737 query = db[table][field]>args[i]
7738 elif tokens[2]=='ge':
7739 query = db[table][field]>=args[i]
7740 elif tokens[2]=='le':
7741 query = db[table][field]<=args[i]
7742 elif tokens[2]=='year':
7743 query = db[table][field].year()==args[i]
7744 elif tokens[2]=='month':
7745 query = db[table][field].month()==args[i]
7746 elif tokens[2]=='day':
7747 query = db[table][field].day()==args[i]
7748 elif tokens[2]=='hour':
7749 query = db[table][field].hour()==args[i]
7750 elif tokens[2]=='minute':
7751 query = db[table][field].minutes()==args[i]
7752 elif tokens[2]=='second':
7753 query = db[table][field].seconds()==args[i]
7754 elif tokens[2]=='startswith':
7755 query = db[table][field].startswith(args[i])
7756 elif tokens[2]=='contains':
7757 query = db[table][field].contains(args[i])
7758 else:
7759 raise RuntimeError("invalid pattern: %s" % pattern)
7760 if len(tokens)==4 and tokens[3]=='not':
7761 query = ~query
7762 elif len(tokens)>=4:
7763 raise RuntimeError("invalid pattern: %s" % pattern)
7764 if not otable and isinstance(queries,dict):
7765 dbset = db(queries[table])
7766 if basequery is not None:
7767 dbset = dbset(basequery)
7768 dbset=dbset(query)
7769 else:
7770 raise RuntimeError("missing relation in pattern: %s" % pattern)
7771 elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
7772 ref = tag[tag.find('[')+1:-1]
7773 if '.' in ref and otable:
7774 table,field = ref.split('.')
7775 selfld = '_id'
7776 if db[table][field].type.startswith('reference '):
7777 refs = [ x.name for x in db[otable] if x.type == db[table][field].type ]
7778 else:
7779 refs = [ x.name for x in db[table]._referenced_by if x.tablename==otable ]
7780 if refs:
7781 selfld = refs[0]
7782 if nested_select:
7783 try:
7784 dbset=db(db[table][field].belongs(dbset._select(db[otable][selfld])))
7785 except ValueError:
7786 return Row({'status':400,'pattern':pattern,
7787 'error':'invalid path','response':None})
7788 else:
7789 items = [item.id for item in dbset.select(db[otable][selfld])]
7790 dbset=db(db[table][field].belongs(items))
7791 else:
7792 table = ref
7793 if not otable and isinstance(queries,dict):
7794 dbset = db(queries[table])
7795 dbset=dbset(db[table])
7796 elif tag==':field' and table:
7797
7798 field = args[i]
7799 if not field in db[table]: break
7800
7801 if not db[table][field].readable:
7802 return Row({'status':418,'pattern':pattern,
7803 'error':'I\'m a teapot','response':None})
7804 try:
7805 distinct = vars.get('distinct', False) == 'True'
7806 offset = long(vars.get('offset',None) or 0)
7807 limits = (offset,long(vars.get('limit',None) or 1000)+offset)
7808 except ValueError:
7809 return Row({'status':400,'error':'invalid limits','response':None})
7810 items = dbset.select(db[table][field], distinct=distinct, limitby=limits)
7811 if items:
7812 return Row({'status':200,'response':items,
7813 'pattern':pattern})
7814 else:
7815 return Row({'status':404,'pattern':pattern,
7816 'error':'no record found','response':None})
7817 elif tag != args[i]:
7818 break
7819 otable = table
7820 i += 1
7821 if i==len(tags) and table:
7822 ofields = vars.get('order',db[table]._id.name).split('|')
7823 try:
7824 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
7825 except (KeyError, AttributeError):
7826 return Row({'status':400,'error':'invalid orderby','response':None})
7827 if exposedfields:
7828 fields = [field for field in db[table] if str(field).split('.')[-1] in exposedfields and field.readable]
7829 else:
7830 fields = [field for field in db[table] if field.readable]
7831 count = dbset.count()
7832 try:
7833 offset = long(vars.get('offset',None) or 0)
7834 limits = (offset,long(vars.get('limit',None) or 1000)+offset)
7835 except ValueError:
7836 return Row({'status':400,'error':'invalid limits','response':None})
7837 if count > limits[1]-limits[0]:
7838 return Row({'status':400,'error':'too many records','response':None})
7839 try:
7840 response = dbset.select(limitby=limits,orderby=orderby,*fields)
7841 except ValueError:
7842 return Row({'status':400,'pattern':pattern,
7843 'error':'invalid path','response':None})
7844 return Row({'status':200,'response':response,
7845 'pattern':pattern,'count':count})
7846 return Row({'status':400,'error':'no matching pattern','response':None})
7847
7848 - def define_table(
7849 self,
7850 tablename,
7851 *fields,
7852 **args
7853 ):
7854 if not fields and 'fields' in args:
7855 fields = args.get('fields',())
7856 if not isinstance(tablename, str):
7857 if isinstance(tablename, unicode):
7858 try:
7859 tablename = str(tablename)
7860 except UnicodeEncodeError:
7861 raise SyntaxError("invalid unicode table name")
7862 else:
7863 raise SyntaxError("missing table name")
7864 elif hasattr(self,tablename) or tablename in self.tables:
7865 if not args.get('redefine',False):
7866 raise SyntaxError('table already defined: %s' % tablename)
7867 elif tablename.startswith('_') or hasattr(self,tablename) or \
7868 REGEX_PYTHON_KEYWORDS.match(tablename):
7869 raise SyntaxError('invalid table name: %s' % tablename)
7870 elif self.check_reserved:
7871 self.check_reserved_keyword(tablename)
7872 else:
7873 invalid_args = set(args)-TABLE_ARGS
7874 if invalid_args:
7875 raise SyntaxError('invalid table "%s" attributes: %s' \
7876 % (tablename,invalid_args))
7877 if self._lazy_tables and not tablename in self._LAZY_TABLES:
7878 self._LAZY_TABLES[tablename] = (tablename,fields,args)
7879 table = None
7880 else:
7881 table = self.lazy_define_table(tablename,*fields,**args)
7882 if not tablename in self.tables:
7883 self.tables.append(tablename)
7884 return table
7885
7886 - def lazy_define_table(
7887 self,
7888 tablename,
7889 *fields,
7890 **args
7891 ):
7892 args_get = args.get
7893 common_fields = self._common_fields
7894 if common_fields:
7895 fields = list(fields) + list(common_fields)
7896
7897 table_class = args_get('table_class',Table)
7898 table = table_class(self, tablename, *fields, **args)
7899 table._actual = True
7900 self[tablename] = table
7901
7902 table._create_references()
7903 for field in table:
7904 if field.requires == DEFAULT:
7905 field.requires = sqlhtml_validators(field)
7906
7907 migrate = self._migrate_enabled and args_get('migrate',self._migrate)
7908 if migrate and not self._uri in (None,'None') \
7909 or self._adapter.dbengine=='google:datastore':
7910 fake_migrate = self._fake_migrate_all or \
7911 args_get('fake_migrate',self._fake_migrate)
7912 polymodel = args_get('polymodel',None)
7913 try:
7914 GLOBAL_LOCKER.acquire()
7915 self._lastsql = self._adapter.create_table(
7916 table,migrate=migrate,
7917 fake_migrate=fake_migrate,
7918 polymodel=polymodel)
7919 finally:
7920 GLOBAL_LOCKER.release()
7921 else:
7922 table._dbt = None
7923 on_define = args_get('on_define',None)
7924 if on_define: on_define(table)
7925 return table
7926
7927 - def as_dict(self, flat=False, sanitize=True):
7928 db_uid = uri = None
7929 if not sanitize:
7930 uri, db_uid = (self._uri, self._db_uid)
7931 db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid,
7932 **dict([(k, getattr(self, "_" + k, None))
7933 for k in 'pool_size','folder','db_codec',
7934 'check_reserved','migrate','fake_migrate',
7935 'migrate_enabled','fake_migrate_all',
7936 'decode_credentials','driver_args',
7937 'adapter_args', 'attempts',
7938 'bigint_id','debug','lazy_tables',
7939 'do_connect']))
7940 for table in self:
7941 db_as_dict["tables"].append(table.as_dict(flat=flat,
7942 sanitize=sanitize))
7943 return db_as_dict
7944
7945 - def as_xml(self, sanitize=True):
7950
7951 - def as_json(self, sanitize=True):
7956
7957 - def as_yaml(self, sanitize=True):
7962
7964 try:
7965 return tablename in self.tables
7966 except AttributeError:
7967
7968 return False
7969
7970 has_key = __contains__
7971
7972 - def get(self,key,default=None):
7973 return self.__dict__.get(key,default)
7974
7976 for tablename in self.tables:
7977 yield self[tablename]
7978
7981
7983 if ogetattr(self,'_lazy_tables') and \
7984 key in ogetattr(self,'_LAZY_TABLES'):
7985 tablename, fields, args = self._LAZY_TABLES.pop(key)
7986 return self.lazy_define_table(tablename,*fields,**args)
7987 return ogetattr(self, key)
7988
7990 osetattr(self, str(key), value)
7991
7993 if key[:1]!='_' and key in self:
7994 raise SyntaxError(
7995 'Object %s exists and cannot be redefined' % key)
7996 osetattr(self,key,value)
7997
7998 __delitem__ = object.__delattr__
7999
8001 if hasattr(self,'_uri'):
8002 return '<DAL uri="%s">' % hide_password(str(self._uri))
8003 else:
8004 return '<DAL db_uid="%s">' % self._db_uid
8005
8008
8009 - def __call__(self, query=None, ignore_common_filters=None):
8010 if isinstance(query,Table):
8011 query = self._adapter.id_query(query)
8012 elif isinstance(query,Field):
8013 query = query!=None
8014 elif isinstance(query, dict):
8015 icf = query.get("ignore_common_filters")
8016 if icf: ignore_common_filters = icf
8017 return Set(self, query, ignore_common_filters=ignore_common_filters)
8018
8021
8024
8026 self._adapter.close()
8027 if self._db_uid in THREAD_LOCAL.db_instances:
8028 db_group = THREAD_LOCAL.db_instances[self._db_uid]
8029 db_group.remove(self)
8030 if not db_group:
8031 del THREAD_LOCAL.db_instances[self._db_uid]
8032
8033 - def executesql(self, query, placeholders=None, as_dict=False,
8034 fields=None, colnames=None):
8035 """
8036 placeholders is optional and will always be None.
8037 If using raw SQL with placeholders, placeholders may be
8038 a sequence of values to be substituted in
8039 or, (if supported by the DB driver), a dictionary with keys
8040 matching named placeholders in your SQL.
8041
8042 Added 2009-12-05 "as_dict" optional argument. Will always be
8043 None when using DAL. If using raw SQL can be set to True
8044 and the results cursor returned by the DB driver will be
8045 converted to a sequence of dictionaries keyed with the db
8046 field names. Tested with SQLite but should work with any database
8047 since the cursor.description used to get field names is part of the
8048 Python dbi 2.0 specs. Results returned with as_dict=True are
8049 the same as those returned when applying .to_list() to a DAL query.
8050
8051 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
8052
8053 Added 2012-08-24 "fields" and "colnames" optional arguments. If either
8054 is provided, the results cursor returned by the DB driver will be
8055 converted to a DAL Rows object using the db._adapter.parse() method.
8056
8057 The "fields" argument is a list of DAL Field objects that match the
8058 fields returned from the DB. The Field objects should be part of one or
8059 more Table objects defined on the DAL object. The "fields" list can
8060 include one or more DAL Table objects in addition to or instead of
8061 including Field objects, or it can be just a single table (not in a
8062 list). In that case, the Field objects will be extracted from the
8063 table(s).
8064
8065 Instead of specifying the "fields" argument, the "colnames" argument
8066 can be specified as a list of field names in tablename.fieldname format.
8067 Again, these should represent tables and fields defined on the DAL
8068 object.
8069
8070 It is also possible to specify both "fields" and the associated
8071 "colnames". In that case, "fields" can also include DAL Expression
8072 objects in addition to Field objects. For Field objects in "fields",
8073 the associated "colnames" must still be in tablename.fieldname format.
8074 For Expression objects in "fields", the associated "colnames" can
8075 be any arbitrary labels.
8076
8077 Note, the DAL Table objects referred to by "fields" or "colnames" can
8078 be dummy tables and do not have to represent any real tables in the
8079 database. Also, note that the "fields" and "colnames" must be in the
8080 same order as the fields in the results cursor returned from the DB.
8081 """
8082 adapter = self._adapter
8083 if placeholders:
8084 adapter.execute(query, placeholders)
8085 else:
8086 adapter.execute(query)
8087 if as_dict:
8088 if not hasattr(adapter.cursor,'description'):
8089 raise RuntimeError("database does not support executesql(...,as_dict=True)")
8090
8091
8092
8093 columns = adapter.cursor.description
8094
8095 fields = [f[0] for f in columns]
8096
8097 data = adapter._fetchall()
8098
8099
8100 return [dict(zip(fields,row)) for row in data]
8101 try:
8102 data = adapter._fetchall()
8103 except:
8104 return None
8105 if fields or colnames:
8106 fields = [] if fields is None else fields
8107 if not isinstance(fields, list):
8108 fields = [fields]
8109 extracted_fields = []
8110 for field in fields:
8111 if isinstance(field, Table):
8112 extracted_fields.extend([f for f in field])
8113 else:
8114 extracted_fields.append(field)
8115 if not colnames:
8116 colnames = ['%s.%s' % (f.tablename, f.name)
8117 for f in extracted_fields]
8118 data = adapter.parse(
8119 data, fields=extracted_fields, colnames=colnames)
8120 return data
8121
8123 for table in self:
8124 table._referenced_by = [field for field in table._referenced_by
8125 if not field.table==thistable]
8126
8128 step = long(kwargs.get('max_fetch_rows,',500))
8129 write_colnames = kwargs['write_colnames'] = \
8130 kwargs.get("write_colnames", True)
8131 for table in self.tables:
8132 ofile.write('TABLE %s\r\n' % table)
8133 query = self._adapter.id_query(self[table])
8134 nrows = self(query).count()
8135 kwargs['write_colnames'] = write_colnames
8136 for k in range(0,nrows,step):
8137 self(query).select(limitby=(k,k+step)).export_to_csv_file(
8138 ofile, *args, **kwargs)
8139 kwargs['write_colnames'] = False
8140 ofile.write('\r\n\r\n')
8141 ofile.write('END')
8142
8143 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
8144 unique='uuid', map_tablenames=None,
8145 ignore_missing_tables=False,
8146 *args, **kwargs):
8147
8148 id_offset = {}
8149 map_tablenames = map_tablenames or {}
8150 for line in ifile:
8151 line = line.strip()
8152 if not line:
8153 continue
8154 elif line == 'END':
8155 return
8156 elif not line.startswith('TABLE ') or \
8157 not line[6:] in self.tables:
8158 raise SyntaxError('invalid file format')
8159 else:
8160 tablename = line[6:]
8161 tablename = map_tablenames.get(tablename,tablename)
8162 if tablename is not None and tablename in self.tables:
8163 self[tablename].import_from_csv_file(
8164 ifile, id_map, null, unique, id_offset,
8165 *args, **kwargs)
8166 elif tablename is None or ignore_missing_tables:
8167
8168 for line in ifile:
8169 if not line.strip():
8170 break
8171 else:
8172 raise RuntimeError("Unable to import table that does not exist.\nTry db.import_from_csv_file(..., map_tablenames={'table':'othertable'},ignore_missing_tables=True)")
8173
8176 return DAL('<zombie>',db_uid=db_uid)
8177
8180
8181 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
8184 """
8185 Helper class providing a comma-separated string having all the field names
8186 (prefixed by table name and '.')
8187
8188 normally only called from within gluon.sql
8189 """
8190
8193
8195 return ', '.join([str(field) for field in self._table])
8196
8199
8201 if not self._record:
8202 self._record = self._table[long(self)]
8203 if not self._record:
8204 raise RuntimeError(
8205 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, long(self)))
8206
8208 if key == 'id':
8209 return long(self)
8210 if key in self._table:
8211 self.__allocate()
8212 if self._record:
8213 return self._record.get(key,None)
8214 else:
8215 return None
8216
8217 - def get(self, key, default=None):
8219
8226
8228 if key == 'id':
8229 return long(self)
8230 self.__allocate()
8231 return self._record.get(key, None)
8232
8234 self.__allocate()
8235 self._record[key] = value
8236
8239 return marshal.loads(data)
8240
8242 try:
8243 marshal_dump = marshal.dumps(long(data))
8244 except AttributeError:
8245 marshal_dump = 'i%s' % struct.pack('<i', long(data))
8246 return (Reference_unpickler, (marshal_dump,))
8247
8248 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
8258 def _decorated(f):
8259 instance = self.table
8260 import types
8261 method = types.MethodType(f, instance, instance.__class__)
8262 name = method_name or f.func_name
8263 setattr(instance, name, method)
8264 return f
8265 return _decorated
8266
8268
8269 """
8270 an instance of this class represents a database table
8271
8272 Example::
8273
8274 db = DAL(...)
8275 db.define_table('users', Field('name'))
8276 db.users.insert(name='me') # print db.users._insert(...) to see SQL
8277 db.users.drop()
8278 """
8279
8280 - def __init__(
8281 self,
8282 db,
8283 tablename,
8284 *fields,
8285 **args
8286 ):
8287 """
8288 Initializes the table and performs checking on the provided fields.
8289
8290 Each table will have automatically an 'id'.
8291
8292 If a field is of type Table, the fields (excluding 'id') from that table
8293 will be used instead.
8294
8295 :raises SyntaxError: when a supplied field is of incorrect type.
8296 """
8297 self._actual = False
8298 self._tablename = tablename
8299 self._ot = args.get('actual_name')
8300 self._sequence_name = args.get('sequence_name') or \
8301 db and db._adapter.sequence_name(tablename)
8302 self._trigger_name = args.get('trigger_name') or \
8303 db and db._adapter.trigger_name(tablename)
8304 self._common_filter = args.get('common_filter')
8305 self._format = args.get('format')
8306 self._singular = args.get(
8307 'singular',tablename.replace('_',' ').capitalize())
8308 self._plural = args.get(
8309 'plural',pluralize(self._singular.lower()).capitalize())
8310
8311 if 'primarykey' in args and args['primarykey'] is not None:
8312 self._primarykey = args.get('primarykey')
8313
8314 self._before_insert = []
8315 self._before_update = [Set.delete_uploaded_files]
8316 self._before_delete = [Set.delete_uploaded_files]
8317 self._after_insert = []
8318 self._after_update = []
8319 self._after_delete = []
8320
8321 self.add_method = MethodAdder(self)
8322
8323 fieldnames,newfields=set(),[]
8324 _primarykey = getattr(self, '_primarykey', None)
8325 if _primarykey is not None:
8326 if not isinstance(_primarykey, list):
8327 raise SyntaxError(
8328 "primarykey must be a list of fields from table '%s'" \
8329 % tablename)
8330 if len(_primarykey)==1:
8331 self._id = [f for f in fields if isinstance(f,Field) \
8332 and f.name==_primarykey[0]][0]
8333 elif not [f for f in fields if (isinstance(f,Field) and
8334 f.type=='id') or (isinstance(f, dict) and
8335 f.get("type", None)=="id")]:
8336 field = Field('id', 'id')
8337 newfields.append(field)
8338 fieldnames.add('id')
8339 self._id = field
8340 virtual_fields = []
8341 def include_new(field):
8342 newfields.append(field)
8343 fieldnames.add(field.name)
8344 if field.type=='id':
8345 self._id = field
8346 for field in fields:
8347 if isinstance(field, (FieldMethod, FieldVirtual)):
8348 virtual_fields.append(field)
8349 elif isinstance(field, Field) and not field.name in fieldnames:
8350 if field.db is not None:
8351 field = copy.copy(field)
8352 include_new(field)
8353 elif isinstance(field, dict) and not field['fieldname'] in fieldnames:
8354 include_new(Field(**field))
8355 elif isinstance(field, Table):
8356 table = field
8357 for field in table:
8358 if not field.name in fieldnames and not field.type=='id':
8359 t2 = not table._actual and self._tablename
8360 include_new(field.clone(point_self_references_to=t2))
8361 elif not isinstance(field, (Field, Table)):
8362 raise SyntaxError(
8363 'define_table argument is not a Field or Table: %s' % field)
8364 fields = newfields
8365 self._db = db
8366 tablename = tablename
8367 self._fields = SQLCallableList()
8368 self.virtualfields = []
8369 fields = list(fields)
8370
8371 if db and db._adapter.uploads_in_blob==True:
8372 uploadfields = [f.name for f in fields if f.type=='blob']
8373 for field in fields:
8374 fn = field.uploadfield
8375 if isinstance(field, Field) and field.type == 'upload'\
8376 and fn is True:
8377 fn = field.uploadfield = '%s_blob' % field.name
8378 if isinstance(fn,str) and not fn in uploadfields:
8379 fields.append(Field(fn,'blob',default='',
8380 writable=False,readable=False))
8381
8382 lower_fieldnames = set()
8383 reserved = dir(Table) + ['fields']
8384 if (db and db.check_reserved):
8385 check_reserved = db.check_reserved_keyword
8386 else:
8387 def check_reserved(field_name):
8388 if field_name in reserved:
8389 raise SyntaxError("field name %s not allowed" % field_name)
8390 for field in fields:
8391 field_name = field.name
8392 check_reserved(field_name)
8393 fn_lower = field_name.lower()
8394 if fn_lower in lower_fieldnames:
8395 raise SyntaxError("duplicate field %s in table %s" \
8396 % (field_name, tablename))
8397 else:
8398 lower_fieldnames.add(fn_lower)
8399
8400 self.fields.append(field_name)
8401 self[field_name] = field
8402 if field.type == 'id':
8403 self['id'] = field
8404 field.tablename = field._tablename = tablename
8405 field.table = field._table = self
8406 field.db = field._db = db
8407 self.ALL = SQLALL(self)
8408
8409 if _primarykey is not None:
8410 for k in _primarykey:
8411 if k not in self.fields:
8412 raise SyntaxError(
8413 "primarykey must be a list of fields from table '%s " % tablename)
8414 else:
8415 self[k].notnull = True
8416 for field in virtual_fields:
8417 self[field.name] = field
8418
8419 @property
8422
8423 - def update(self,*args,**kwargs):
8424 raise RuntimeError("Syntax Not Supported")
8425
8426 - def _enable_record_versioning(self,
8427 archive_db=None,
8428 archive_name = '%(tablename)s_archive',
8429 is_active = 'is_active',
8430 current_record = 'current_record',
8431 current_record_label = None):
8432 db = self._db
8433 archive_db = archive_db or db
8434 archive_name = archive_name % dict(tablename=self._tablename)
8435 if archive_name in archive_db.tables():
8436 return
8437 fieldnames = self.fields()
8438 same_db = archive_db is db
8439 field_type = self if same_db else 'bigint'
8440 clones = []
8441 for field in self:
8442 nfk = same_db or not field.type.startswith('reference')
8443 clones.append(field.clone(
8444 unique=False, type=field.type if nfk else 'bigint'))
8445 archive_db.define_table(
8446 archive_name, Field(current_record,field_type,
8447 label=current_record_label), *clones)
8448 self._before_update.append(
8449 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record:
8450 archive_record(qset,fs,db[an],cn))
8451 if is_active and is_active in fieldnames:
8452 self._before_delete.append(
8453 lambda qset: qset.update(is_active=False))
8454 newquery = lambda query, t=self, name=self._tablename: \
8455 reduce(AND,[db[tn].is_active == True
8456 for tn in db._adapter.tables(query)
8457 if tn==name or getattr(db[tn],'_ot',None)==name])
8458 query = self._common_filter
8459 if query:
8460 newquery = query & newquery
8461 self._common_filter = newquery
8462
8470
8472 db = self._db
8473 pr = db._pending_references
8474 self._referenced_by = []
8475 self._references = []
8476 for field in self:
8477 fieldname = field.name
8478 field_type = field.type
8479 if isinstance(field_type,str) and field_type[:10] == 'reference ':
8480 ref = field_type[10:].strip()
8481 if not ref:
8482 SyntaxError('Table: reference to nothing: %s' %ref)
8483 if '.' in ref:
8484 rtablename, throw_it,rfieldname = ref.partition('.')
8485 else:
8486 rtablename, rfieldname = ref, None
8487 if not rtablename in db:
8488 pr[rtablename] = pr.get(rtablename,[]) + [field]
8489 continue
8490 rtable = db[rtablename]
8491 if rfieldname:
8492 if not hasattr(rtable,'_primarykey'):
8493 raise SyntaxError(
8494 'keyed tables can only reference other keyed tables (for now)')
8495 if rfieldname not in rtable.fields:
8496 raise SyntaxError(
8497 "invalid field '%s' for referenced table '%s' in table '%s'" \
8498 % (rfieldname, rtablename, self._tablename))
8499 rfield = rtable[rfieldname]
8500 else:
8501 rfield = rtable._id
8502 rtable._referenced_by.append(field)
8503 field.referent = rfield
8504 self._references.append(field)
8505 else:
8506 field.referent = None
8507 for referee in pr.get(self._tablename,[]):
8508 self._referenced_by.append(referee)
8509
8511 return dict([(k, v) for (k, v) in record.iteritems() if k
8512 in self.fields and (self[k].type!='id' or id)])
8513
8515 """ for keyed table only """
8516 query = None
8517 for k,v in key.iteritems():
8518 if k in self._primarykey:
8519 if query:
8520 query = query & (self[k] == v)
8521 else:
8522 query = (self[k] == v)
8523 else:
8524 raise SyntaxError(
8525 'Field %s is not part of the primary key of %s' % \
8526 (k,self._tablename))
8527 return query
8528
8530 if not key:
8531 return None
8532 elif isinstance(key, dict):
8533 """ for keyed table """
8534 query = self._build_query(key)
8535 return self._db(query).select(limitby=(0,1), orderby_on_limitby=False).first()
8536 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key):
8537 return self._db(self._id == key).select(limitby=(0,1), orderby_on_limitby=False).first()
8538 elif key:
8539 return ogetattr(self, str(key))
8540
8542 for_update = kwargs.get('_for_update',False)
8543 if '_for_update' in kwargs: del kwargs['_for_update']
8544
8545 orderby = kwargs.get('_orderby',None)
8546 if '_orderby' in kwargs: del kwargs['_orderby']
8547
8548 if not key is DEFAULT:
8549 if isinstance(key, Query):
8550 record = self._db(key).select(
8551 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8552 elif not str(key).isdigit():
8553 record = None
8554 else:
8555 record = self._db(self._id == key).select(
8556 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8557 if record:
8558 for k,v in kwargs.iteritems():
8559 if record[k]!=v: return None
8560 return record
8561 elif kwargs:
8562 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
8563 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8564 else:
8565 return None
8566
8568 if isinstance(key, dict) and isinstance(value, dict):
8569 """ option for keyed table """
8570 if set(key.keys()) == set(self._primarykey):
8571 value = self._filter_fields(value)
8572 kv = {}
8573 kv.update(value)
8574 kv.update(key)
8575 if not self.insert(**kv):
8576 query = self._build_query(key)
8577 self._db(query).update(**self._filter_fields(value))
8578 else:
8579 raise SyntaxError(
8580 'key must have all fields from primary key: %s'%\
8581 (self._primarykey))
8582 elif str(key).isdigit():
8583 if key == 0:
8584 self.insert(**self._filter_fields(value))
8585 elif self._db(self._id == key)\
8586 .update(**self._filter_fields(value)) is None:
8587 raise SyntaxError('No such record: %s' % key)
8588 else:
8589 if isinstance(key, dict):
8590 raise SyntaxError(
8591 'value must be a dictionary: %s' % value)
8592 osetattr(self, str(key), value)
8593
8594 __getattr__ = __getitem__
8595
8597 if key[:1]!='_' and key in self:
8598 raise SyntaxError('Object exists and cannot be redefined: %s' % key)
8599 osetattr(self,key,value)
8600
8602 if isinstance(key, dict):
8603 query = self._build_query(key)
8604 if not self._db(query).delete():
8605 raise SyntaxError('No such record: %s' % key)
8606 elif not str(key).isdigit() or \
8607 not self._db(self._id == key).delete():
8608 raise SyntaxError('No such record: %s' % key)
8609
8611 return hasattr(self,key)
8612
8613 has_key = __contains__
8614
8616 return self.__dict__.items()
8617
8619 for fieldname in self.fields:
8620 yield self[fieldname]
8621
8624
8625
8627 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
8628
8630 if self._ot is not None:
8631 ot = self._db._adapter.QUOTE_TEMPLATE % self._ot
8632 if 'Oracle' in str(type(self._db._adapter)):
8633 return '%s %s' % (ot, self._tablename)
8634 return '%s AS %s' % (ot, self._tablename)
8635 return self._tablename
8636
8637 - def _drop(self, mode = ''):
8638 return self._db._adapter._drop(self, mode)
8639
8640 - def drop(self, mode = ''):
8641 return self._db._adapter.drop(self,mode)
8642
8643 - def _listify(self,fields,update=False):
8644 new_fields = {}
8645
8646
8647 for name in fields:
8648 if not name in self.fields:
8649 if name != 'id':
8650 raise SyntaxError(
8651 'Field %s does not belong to the table' % name)
8652 else:
8653 field = self[name]
8654 value = fields[name]
8655 if field.filter_in:
8656 value = field.filter_in(value)
8657 new_fields[name] = (field,value)
8658
8659
8660 to_compute = []
8661 for ofield in self:
8662 name = ofield.name
8663 if not name in new_fields:
8664
8665 if ofield.compute:
8666 to_compute.append((name,ofield))
8667
8668 elif not update and not ofield.default is None:
8669 value = ofield.default
8670 fields[name] = value
8671 new_fields[name] = (ofield,value)
8672
8673 elif update and not ofield.update is None:
8674 value = ofield.update
8675 fields[name] = value
8676 new_fields[name] = (ofield,value)
8677
8678 elif not update and ofield.required:
8679 raise RuntimeError(
8680 'Table: missing required field: %s' % name)
8681
8682 if to_compute:
8683 row = Row(fields)
8684 for name,ofield in to_compute:
8685
8686 try:
8687 row[name] = new_value = ofield.compute(row)
8688 new_fields[name] = (ofield, new_value)
8689 except (KeyError, AttributeError):
8690
8691 if ofield.required:
8692 raise SyntaxError('unable to compute field: %s' % name)
8693 return new_fields.values()
8694
8696 for field in self:
8697 if field.type=='upload' and field.name in fields:
8698 value = fields[field.name]
8699 if value is not None and not isinstance(value,str):
8700 if hasattr(value,'file') and hasattr(value,'filename'):
8701 new_name = field.store(value.file,filename=value.filename)
8702 elif hasattr(value,'read') and hasattr(value,'name'):
8703 new_name = field.store(value,filename=value.name)
8704 else:
8705 raise RuntimeError("Unable to handle upload")
8706 fields[field.name] = new_name
8707
8709 "If there are no fields/values specified, return table defaults"
8710 if not fields:
8711 fields = {}
8712 for field in self:
8713 if field.type != "id":
8714 fields[field.name] = field.default
8715 return fields
8716
8720
8730
8746
8772
8774 if _key is DEFAULT:
8775 record = self(**values)
8776 elif isinstance(_key,dict):
8777 record = self(**_key)
8778 else:
8779 record = self(_key)
8780 if record:
8781 record.update_record(**values)
8782 newid = None
8783 else:
8784 newid = self.insert(**values)
8785 return newid
8786
8788 """
8789 here items is a list of dictionaries
8790 """
8791 items = [self._listify(item) for item in items]
8792 if any(f(item) for item in items for f in self._before_insert):return 0
8793 ret = self._db._adapter.bulk_insert(self,items)
8794 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert]
8795 return ret
8796
8798 return self._db._adapter._truncate(self, mode)
8799
8801 return self._db._adapter.truncate(self, mode)
8802
8803 - def import_from_csv_file(
8804 self,
8805 csvfile,
8806 id_map=None,
8807 null='<NULL>',
8808 unique='uuid',
8809 id_offset=None,
8810 *args, **kwargs
8811 ):
8812 """
8813 Import records from csv file.
8814 Column headers must have same names as table fields.
8815 Field 'id' is ignored.
8816 If column names read 'table.file' the 'table.' prefix is ignored.
8817 'unique' argument is a field which must be unique
8818 (typically a uuid field)
8819 'restore' argument is default False;
8820 if set True will remove old values in table first.
8821 'id_map' ff set to None will not map ids.
8822 The import will keep the id numbers in the restored table.
8823 This assumes that there is an field of type id that
8824 is integer and in incrementing order.
8825 Will keep the id numbers in restored table.
8826 """
8827
8828 delimiter = kwargs.get('delimiter', ',')
8829 quotechar = kwargs.get('quotechar', '"')
8830 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
8831 restore = kwargs.get('restore', False)
8832 if restore:
8833 self._db[self].truncate()
8834
8835 reader = csv.reader(csvfile, delimiter=delimiter,
8836 quotechar=quotechar, quoting=quoting)
8837 colnames = None
8838 if isinstance(id_map, dict):
8839 if not self._tablename in id_map:
8840 id_map[self._tablename] = {}
8841 id_map_self = id_map[self._tablename]
8842
8843 def fix(field, value, id_map, id_offset):
8844 list_reference_s='list:reference'
8845 if value == null:
8846 value = None
8847 elif field.type=='blob':
8848 value = base64.b64decode(value)
8849 elif field.type=='double' or field.type=='float':
8850 if not value.strip():
8851 value = None
8852 else:
8853 value = float(value)
8854 elif field.type in ('integer','bigint'):
8855 if not value.strip():
8856 value = None
8857 else:
8858 value = long(value)
8859 elif field.type.startswith('list:string'):
8860 value = bar_decode_string(value)
8861 elif field.type.startswith(list_reference_s):
8862 ref_table = field.type[len(list_reference_s):].strip()
8863 if id_map is not None:
8864 value = [id_map[ref_table][long(v)] \
8865 for v in bar_decode_string(value)]
8866 else:
8867 value = [v for v in bar_decode_string(value)]
8868 elif field.type.startswith('list:'):
8869 value = bar_decode_integer(value)
8870 elif id_map and field.type.startswith('reference'):
8871 try:
8872 value = id_map[field.type[9:].strip()][long(value)]
8873 except KeyError:
8874 pass
8875 elif id_offset and field.type.startswith('reference'):
8876 try:
8877 value = id_offset[field.type[9:].strip()]+long(value)
8878 except KeyError:
8879 pass
8880 return (field.name, value)
8881
8882 def is_id(colname):
8883 if colname in self:
8884 return self[colname].type == 'id'
8885 else:
8886 return False
8887
8888 first = True
8889 unique_idx = None
8890 for lineno, line in enumerate(reader):
8891 if not line:
8892 break
8893 if not colnames:
8894
8895 colnames = [x.split('.',1)[-1] for x in line][:len(line)]
8896 cols, cid = [], None
8897 for i,colname in enumerate(colnames):
8898 if is_id(colname):
8899 cid = i
8900 elif colname in self.fields:
8901 cols.append((i,self[colname]))
8902 if colname == unique:
8903 unique_idx = i
8904 else:
8905
8906 items = []
8907 for i, field in cols:
8908 try:
8909 items.append(fix(field, line[i], id_map, id_offset))
8910 except ValueError:
8911 raise RuntimeError("Unable to parse line:%s field:%s value:'%s'"
8912 % (lineno+1,field,line[i]))
8913
8914 if not (id_map or cid is None or id_offset is None or unique_idx):
8915 csv_id = long(line[cid])
8916 curr_id = self.insert(**dict(items))
8917 if first:
8918 first = False
8919
8920
8921
8922 id_offset[self._tablename] = (curr_id-csv_id) \
8923 if curr_id>csv_id else 0
8924
8925 while curr_id<csv_id+id_offset[self._tablename]:
8926 self._db(self._db[self][colnames[cid]] == curr_id).delete()
8927 curr_id = self.insert(**dict(items))
8928
8929
8930 elif not unique_idx:
8931 new_id = self.insert(**dict(items))
8932 else:
8933 unique_value = line[unique_idx]
8934 query = self._db[self][unique] == unique_value
8935 record = self._db(query).select().first()
8936 if record:
8937 record.update_record(**dict(items))
8938 new_id = record[self._id.name]
8939 else:
8940 new_id = self.insert(**dict(items))
8941 if id_map and cid is not None:
8942 id_map_self[long(line[cid])] = new_id
8943
8944 - def as_dict(self, flat=False, sanitize=True):
8956
8957 - def as_xml(self, sanitize=True):
8962
8963 - def as_json(self, sanitize=True):
8968
8969 - def as_yaml(self, sanitize=True):
8974
8977
8978 - def on(self, query):
8979 return Expression(self._db,self._db._adapter.ON,self,query)
8980
8982 tablenames = qset.db._adapter.tables(qset.query)
8983 if len(tablenames)!=1: raise RuntimeError("cannot update join")
8984 table = qset.db[tablenames[0]]
8985 for row in qset.select():
8986 fields = archive_table._filter_fields(row)
8987 fields[current_record] = row.id
8988 archive_table.insert(**fields)
8989 return False
8990
8994
8995 - def __init__(
8996 self,
8997 db,
8998 op,
8999 first=None,
9000 second=None,
9001 type=None,
9002 **optional_args
9003 ):
9004
9005 self.db = db
9006 self.op = op
9007 self.first = first
9008 self.second = second
9009 self._table = getattr(first,'_table',None)
9010
9011 if not type and first and hasattr(first,'type'):
9012 self.type = first.type
9013 else:
9014 self.type = type
9015 self.optional_args = optional_args
9016
9020
9024
9028
9032
9036
9040
9044
9048
9052
9056
9060
9064
9068
9072
9076
9080
9084
9088
9090 db = self.db
9091 if start < 0:
9092 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
9093 else:
9094 pos0 = start + 1
9095
9096 if stop < 0:
9097 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
9098 elif stop == sys.maxint:
9099 length = self.len()
9100 else:
9101 length = '(%s - %s)' % (stop + 1, pos0)
9102 return Expression(db,db._adapter.SUBSTRING,
9103 self, (pos0, length), self.type)
9104
9106 return self[i:i + 1]
9107
9109 return self.db._adapter.expand(self,self.type)
9110
9112 db = self.db
9113 return Expression(db,db._adapter.COMMA,self,other,self.type)
9114
9120
9124
9126 db = self.db
9127 if self.type in ('integer','bigint'):
9128 result_type = 'integer'
9129 elif self.type in ['date','time','datetime','double','float']:
9130 result_type = 'double'
9131 elif self.type.startswith('decimal('):
9132 result_type = self.type
9133 else:
9134 raise SyntaxError("subtraction operation not supported for type")
9135 return Expression(db,db._adapter.SUB,self,other,result_type)
9136
9140
9144
9148
9152
9156
9160
9164
9168
9172
9173 - def like(self, value, case_sensitive=False):
9174 db = self.db
9175 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE
9176 return Query(db, op, self, value)
9177
9181
9182 - def belongs(self, *value, **kwattr):
9183 """
9184 Accepts the following inputs:
9185 field.belongs(1,2)
9186 field.belongs((1,2))
9187 field.belongs(query)
9188
9189 Does NOT accept:
9190 field.belongs(1)
9191 """
9192 db = self.db
9193 if len(value) == 1:
9194 value = value[0]
9195 if isinstance(value,Query):
9196 value = db(value)._select(value.first._table._id)
9197 elif not isinstance(value, basestring):
9198 value = set(value)
9199 if kwattr.get('null') and None in value:
9200 value.remove(None)
9201 return (self == None) | Query(db, db._adapter.BELONGS, self, value)
9202 return Query(db, db._adapter.BELONGS, self, value)
9203
9205 db = self.db
9206 if not self.type in ('string', 'text', 'json'):
9207 raise SyntaxError("startswith used with incompatible field type")
9208 return Query(db, db._adapter.STARTSWITH, self, value)
9209
9211 db = self.db
9212 if not self.type in ('string', 'text', 'json'):
9213 raise SyntaxError("endswith used with incompatible field type")
9214 return Query(db, db._adapter.ENDSWITH, self, value)
9215
9216 - def contains(self, value, all=False, case_sensitive=False):
9217 """
9218 The case_sensitive parameters is only useful for PostgreSQL
9219 For other RDMBs it is ignored and contains is always case in-sensitive
9220 For MongoDB and GAE contains is always case sensitive
9221 """
9222 db = self.db
9223 if isinstance(value,(list, tuple)):
9224 subqueries = [self.contains(str(v).strip(),case_sensitive=case_sensitive)
9225 for v in value if str(v).strip()]
9226 if not subqueries:
9227 return self.contains('')
9228 else:
9229 return reduce(all and AND or OR,subqueries)
9230 if not self.type in ('string', 'text', 'json') and not self.type.startswith('list:'):
9231 raise SyntaxError("contains used with incompatible field type")
9232 return Query(db, db._adapter.CONTAINS, self, value, case_sensitive=case_sensitive)
9233
9237
9238
9239
9240 - def st_asgeojson(self, precision=15, options=0, version=1):
9244
9245 - def st_astext(self):
9246 db = self.db
9247 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
9248
9252
9256
9260
9264
9265
9266
9270
9274
9278
9282
9286
9290
9295 """
9296 allows defining of custom SQL types
9297
9298 Example::
9299
9300 decimal = SQLCustomType(
9301 type ='double',
9302 native ='integer',
9303 encoder =(lambda x: int(float(x) * 100)),
9304 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
9305 )
9306
9307 db.define_table(
9308 'example',
9309 Field('value', type=decimal)
9310 )
9311
9312 :param type: the web2py type (default = 'string')
9313 :param native: the backend type
9314 :param encoder: how to encode the value to store it in the backend
9315 :param decoder: how to decode the value retrieved from the backend
9316 :param validator: what validators to use ( default = None, will use the
9317 default validator for type)
9318 """
9319
9320 - def __init__(
9321 self,
9322 type='string',
9323 native=None,
9324 encoder=None,
9325 decoder=None,
9326 validator=None,
9327 _class=None,
9328 ):
9329
9330 self.type = type
9331 self.native = native
9332 self.encoder = encoder or (lambda x: x)
9333 self.decoder = decoder or (lambda x: x)
9334 self.validator = validator
9335 self._class = _class or type
9336
9338 try:
9339 return self.type.startswith(self, text)
9340 except TypeError:
9341 return False
9342
9345
9348
9351
9353 - def __init__(self, name, f=None, ftype='string',label=None,table_name=None):
9368 return '%s.%s' % (self.tablename, self.name)
9369
9371 - def __init__(self, name, f=None, handler=None):
9375
9377 return ', '.join(str(y) for y in x or [])
9378
9379 -class Field(Expression):
9380
9381 Virtual = FieldVirtual
9382 Method = FieldMethod
9383 Lazy = FieldMethod
9384
9385 """
9386 an instance of this class represents a database field
9387
9388 example::
9389
9390 a = Field(name, 'string', length=32, default=None, required=False,
9391 requires=IS_NOT_EMPTY(), ondelete='CASCADE',
9392 notnull=False, unique=False,
9393 uploadfield=True, widget=None, label=None, comment=None,
9394 uploadfield=True, # True means store on disk,
9395 # 'a_field_name' means store in this field in db
9396 # False means file content will be discarded.
9397 writable=True, readable=True, update=None, authorize=None,
9398 autodelete=False, represent=None, uploadfolder=None,
9399 uploadseparate=False # upload to separate directories by uuid_keys
9400 # first 2 character and tablename.fieldname
9401 # False - old behavior
9402 # True - put uploaded file in
9403 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
9404 # directory)
9405 uploadfs=None # a pyfilesystem where to store upload
9406
9407 to be used as argument of DAL.define_table
9408
9409 allowed field types:
9410 string, boolean, integer, double, text, blob,
9411 date, time, datetime, upload, password
9412
9413 """
9414
9415 - def __init__(
9416 self,
9417 fieldname,
9418 type='string',
9419 length=None,
9420 default=DEFAULT,
9421 required=False,
9422 requires=DEFAULT,
9423 ondelete='CASCADE',
9424 notnull=False,
9425 unique=False,
9426 uploadfield=True,
9427 widget=None,
9428 label=None,
9429 comment=None,
9430 writable=True,
9431 readable=True,
9432 update=None,
9433 authorize=None,
9434 autodelete=False,
9435 represent=None,
9436 uploadfolder=None,
9437 uploadseparate=False,
9438 uploadfs=None,
9439 compute=None,
9440 custom_store=None,
9441 custom_retrieve=None,
9442 custom_retrieve_file_properties=None,
9443 custom_delete=None,
9444 filter_in = None,
9445 filter_out = None,
9446 custom_qualifier = None,
9447 map_none = None,
9448 ):
9449 self._db = self.db = None
9450 self.op = None
9451 self.first = None
9452 self.second = None
9453 if isinstance(fieldname, unicode):
9454 try:
9455 fieldname = str(fieldname)
9456 except UnicodeEncodeError:
9457 raise SyntaxError('Field: invalid unicode field name')
9458 self.name = fieldname = cleanup(fieldname)
9459 if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \
9460 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
9461 raise SyntaxError('Field: invalid field name: %s' % fieldname)
9462 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
9463 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512)
9464 self.default = default if default!=DEFAULT else (update or None)
9465 self.required = required
9466 self.ondelete = ondelete.upper()
9467 self.notnull = notnull
9468 self.unique = unique
9469 self.uploadfield = uploadfield
9470 self.uploadfolder = uploadfolder
9471 self.uploadseparate = uploadseparate
9472 self.uploadfs = uploadfs
9473 self.widget = widget
9474 self.comment = comment
9475 self.writable = writable
9476 self.readable = readable
9477 self.update = update
9478 self.authorize = authorize
9479 self.autodelete = autodelete
9480 self.represent = list_represent if \
9481 represent==None and type in ('list:integer','list:string') else represent
9482 self.compute = compute
9483 self.isattachment = True
9484 self.custom_store = custom_store
9485 self.custom_retrieve = custom_retrieve
9486 self.custom_retrieve_file_properties = custom_retrieve_file_properties
9487 self.custom_delete = custom_delete
9488 self.filter_in = filter_in
9489 self.filter_out = filter_out
9490 self.custom_qualifier = custom_qualifier
9491 self.label = label if label!=None else fieldname.replace('_',' ').title()
9492 self.requires = requires if requires!=None else []
9493 self.map_none = map_none
9494
9496 self.__dict__.update(*args,**attributes)
9497
9498 - def clone(self,point_self_references_to=False,**args):
9499 field = copy.copy(self)
9500 if point_self_references_to and \
9501 field.type == 'reference %s'+field._tablename:
9502 field.type = 'reference %s' % point_self_references_to
9503 field.__dict__.update(args)
9504 return field
9505
9506 - def store(self, file, filename=None, path=None):
9507 if self.custom_store:
9508 return self.custom_store(file,filename,path)
9509 if isinstance(file, cgi.FieldStorage):
9510 filename = filename or file.filename
9511 file = file.file
9512 elif not filename:
9513 filename = file.name
9514 filename = os.path.basename(filename.replace('/', os.sep)\
9515 .replace('\\', os.sep))
9516 m = REGEX_STORE_PATTERN.search(filename)
9517 extension = m and m.group('e') or 'txt'
9518 uuid_key = web2py_uuid().replace('-', '')[-16:]
9519 encoded_filename = base64.b16encode(filename).lower()
9520 newfilename = '%s.%s.%s.%s' % \
9521 (self._tablename, self.name, uuid_key, encoded_filename)
9522 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
9523 self_uploadfield = self.uploadfield
9524 if isinstance(self_uploadfield,Field):
9525 blob_uploadfield_name = self_uploadfield.uploadfield
9526 keys={self_uploadfield.name: newfilename,
9527 blob_uploadfield_name: file.read()}
9528 self_uploadfield.table.insert(**keys)
9529 elif self_uploadfield == True:
9530 if path:
9531 pass
9532 elif self.uploadfolder:
9533 path = self.uploadfolder
9534 elif self.db._adapter.folder:
9535 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9536 else:
9537 raise RuntimeError(
9538 "you must specify a Field(...,uploadfolder=...)")
9539 if self.uploadseparate:
9540 if self.uploadfs:
9541 raise RuntimeError("not supported")
9542 path = pjoin(path,"%s.%s" %(self._tablename, self.name),
9543 uuid_key[:2])
9544 if not exists(path):
9545 os.makedirs(path)
9546 pathfilename = pjoin(path, newfilename)
9547 if self.uploadfs:
9548 dest_file = self.uploadfs.open(newfilename, 'wb')
9549 else:
9550 dest_file = open(pathfilename, 'wb')
9551 try:
9552 shutil.copyfileobj(file, dest_file)
9553 except IOError:
9554 raise IOError(
9555 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename)
9556 dest_file.close()
9557 return newfilename
9558
9559 - def retrieve(self, name, path=None, nameonly=False):
9560 """
9561 if nameonly==True return (filename, fullfilename) instead of
9562 (filename, stream)
9563 """
9564 self_uploadfield = self.uploadfield
9565 if self.custom_retrieve:
9566 return self.custom_retrieve(name, path)
9567 import http
9568 if self.authorize or isinstance(self_uploadfield, str):
9569 row = self.db(self == name).select().first()
9570 if not row:
9571 raise http.HTTP(404)
9572 if self.authorize and not self.authorize(row):
9573 raise http.HTTP(403)
9574 file_properties = self.retrieve_file_properties(name,path)
9575 filename = file_properties['filename']
9576 if isinstance(self_uploadfield, str):
9577 stream = StringIO.StringIO(row[self_uploadfield] or '')
9578 elif isinstance(self_uploadfield,Field):
9579 blob_uploadfield_name = self_uploadfield.uploadfield
9580 query = self_uploadfield == name
9581 data = self_uploadfield.table(query)[blob_uploadfield_name]
9582 stream = StringIO.StringIO(data)
9583 elif self.uploadfs:
9584
9585 stream = self.uploadfs.open(name, 'rb')
9586 else:
9587
9588
9589
9590 fullname = pjoin(file_properties['path'],name)
9591 if nameonly:
9592 return (filename, fullname)
9593 stream = open(fullname,'rb')
9594 return (filename, stream)
9595
9597 m = REGEX_UPLOAD_PATTERN.match(name)
9598 if not m or not self.isattachment:
9599 raise TypeError('Can\'t retrieve %s file properties' % name)
9600 self_uploadfield = self.uploadfield
9601 if self.custom_retrieve_file_properties:
9602 return self.custom_retrieve_file_properties(name, path)
9603 if m.group('name'):
9604 try:
9605 filename = base64.b16decode(m.group('name'), True)
9606 filename = REGEX_CLEANUP_FN.sub('_', filename)
9607 except (TypeError, AttributeError):
9608 filename = name
9609 else:
9610 filename = name
9611
9612 if isinstance(self_uploadfield, (str, Field)):
9613 return dict(path=None,filename=filename)
9614
9615 if not path:
9616 if self.uploadfolder:
9617 path = self.uploadfolder
9618 else:
9619 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9620 if self.uploadseparate:
9621 t = m.group('table')
9622 f = m.group('field')
9623 u = m.group('uuidkey')
9624 path = pjoin(path,"%s.%s" % (t,f),u[:2])
9625 return dict(path=path,filename=filename)
9626
9627
9643
9655
9656 - def count(self, distinct=None):
9658
9659 - def as_dict(self, flat=False, sanitize=True):
9660 attrs = ("name", 'authorize', 'represent', 'ondelete',
9661 'custom_store', 'autodelete', 'custom_retrieve',
9662 'filter_out', 'uploadseparate', 'widget', 'uploadfs',
9663 'update', 'custom_delete', 'uploadfield', 'uploadfolder',
9664 'custom_qualifier', 'unique', 'writable', 'compute',
9665 'map_none', 'default', 'type', 'required', 'readable',
9666 'requires', 'comment', 'label', 'length', 'notnull',
9667 'custom_retrieve_file_properties', 'filter_in')
9668 serializable = (int, long, basestring, float, tuple,
9669 bool, type(None))
9670
9671 def flatten(obj):
9672 if isinstance(obj, dict):
9673 return dict((flatten(k), flatten(v)) for k, v in
9674 obj.items())
9675 elif isinstance(obj, (tuple, list, set)):
9676 return [flatten(v) for v in obj]
9677 elif isinstance(obj, serializable):
9678 return obj
9679 elif isinstance(obj, (datetime.datetime,
9680 datetime.date, datetime.time)):
9681 return str(obj)
9682 else:
9683 return None
9684
9685 d = dict()
9686 if not (sanitize and not (self.readable or self.writable)):
9687 for attr in attrs:
9688 if flat:
9689 d.update({attr: flatten(getattr(self, attr))})
9690 else:
9691 d.update({attr: getattr(self, attr)})
9692 d["fieldname"] = d.pop("name")
9693 return d
9694
9695 - def as_xml(self, sanitize=True):
9702
9703 - def as_json(self, sanitize=True):
9710
9711 - def as_yaml(self, sanitize=True):
9717
9720
9722 try:
9723 return '%s.%s' % (self.tablename, self.name)
9724 except:
9725 return '<no table>.%s' % self.name
9726
9727
9728 -class Query(object):
9729
9730 """
9731 a query object necessary to define a set.
9732 it can be stored or can be passed to DAL.__call__() to obtain a Set
9733
9734 Example::
9735
9736 query = db.users.name=='Max'
9737 set = db(query)
9738 records = set.select()
9739
9740 """
9741
9742 - def __init__(
9743 self,
9744 db,
9745 op,
9746 first=None,
9747 second=None,
9748 ignore_common_filters = False,
9749 **optional_args
9750 ):
9751 self.db = self._db = db
9752 self.op = op
9753 self.first = first
9754 self.second = second
9755 self.ignore_common_filters = ignore_common_filters
9756 self.optional_args = optional_args
9757
9760
9762 return self.db._adapter.expand(self)
9763
9766
9767 __rand__ = __and__
9768
9770 return Query(self.db,self.db._adapter.OR,self,other)
9771
9772 __ror__ = __or__
9773
9775 if self.op==self.db._adapter.NOT:
9776 return self.first
9777 return Query(self.db,self.db._adapter.NOT,self)
9778
9780 return repr(self) == repr(other)
9781
9783 return not (self == other)
9784
9785 - def case(self,t=1,f=0):
9786 return self.db._adapter.CASE(self,t,f)
9787
9788 - def as_dict(self, flat=False, sanitize=True):
9789 """Experimental stuff
9790
9791 This allows to return a plain dictionary with the basic
9792 query representation. Can be used with json/xml services
9793 for client-side db I/O
9794
9795 Example:
9796 >>> q = db.auth_user.id != 0
9797 >>> q.as_dict(flat=True)
9798 {"op": "NE", "first":{"tablename": "auth_user",
9799 "fieldname": "id"},
9800 "second":0}
9801 """
9802
9803 SERIALIZABLE_TYPES = (tuple, dict, set, list, int, long, float,
9804 basestring, type(None), bool)
9805 def loop(d):
9806 newd = dict()
9807 for k, v in d.items():
9808 if k in ("first", "second"):
9809 if isinstance(v, self.__class__):
9810 newd[k] = loop(v.__dict__)
9811 elif isinstance(v, Field):
9812 newd[k] = {"tablename": v._tablename,
9813 "fieldname": v.name}
9814 elif isinstance(v, Expression):
9815 newd[k] = loop(v.__dict__)
9816 elif isinstance(v, SERIALIZABLE_TYPES):
9817 newd[k] = v
9818 elif isinstance(v, (datetime.date,
9819 datetime.time,
9820 datetime.datetime)):
9821 newd[k] = unicode(v)
9822 elif k == "op":
9823 if callable(v):
9824 newd[k] = v.__name__
9825 elif isinstance(v, basestring):
9826 newd[k] = v
9827 else: pass
9828 elif isinstance(v, SERIALIZABLE_TYPES):
9829 if isinstance(v, dict):
9830 newd[k] = loop(v)
9831 else: newd[k] = v
9832 return newd
9833
9834 if flat:
9835 return loop(self.__dict__)
9836 else: return self.__dict__
9837
9838
9839 - def as_xml(self, sanitize=True):
9846
9847 - def as_json(self, sanitize=True):
9854
9856 if not orderby:
9857 return None
9858 orderby2 = orderby[0]
9859 for item in orderby[1:]:
9860 orderby2 = orderby2 | item
9861 return orderby2
9862
9864 return (query and hasattr(query,'ignore_common_filters') and \
9865 not query.ignore_common_filters)
9866
9868
9869 """
9870 a Set represents a set of records in the database,
9871 the records are identified by the query=Query(...) object.
9872 normally the Set is generated by DAL.__call__(Query(...))
9873
9874 given a set, for example
9875 set = db(db.users.name=='Max')
9876 you can:
9877 set.update(db.users.name='Massimo')
9878 set.delete() # all elements in the set
9879 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
9880 and take subsets:
9881 subset = set(db.users.id<5)
9882 """
9883
9884 - def __init__(self, db, query, ignore_common_filters = None):
9885 self.db = db
9886 self._db = db
9887 self.dquery = None
9888
9889
9890 if isinstance(query, dict):
9891 query = self.parse(query)
9892
9893 if not ignore_common_filters is None and \
9894 use_common_filters(query) == ignore_common_filters:
9895 query = copy.copy(query)
9896 query.ignore_common_filters = ignore_common_filters
9897 self.query = query
9898
9901
9902 - def __call__(self, query, ignore_common_filters=False):
9903 if query is None:
9904 return self
9905 elif isinstance(query,Table):
9906 query = self.db._adapter.id_query(query)
9907 elif isinstance(query,str):
9908 query = Expression(self.db,query)
9909 elif isinstance(query,Field):
9910 query = query!=None
9911 if self.query:
9912 return Set(self.db, self.query & query,
9913 ignore_common_filters=ignore_common_filters)
9914 else:
9915 return Set(self.db, query,
9916 ignore_common_filters=ignore_common_filters)
9917
9918 - def _count(self,distinct=None):
9919 return self.db._adapter._count(self.query,distinct)
9920
9921 - def _select(self, *fields, **attributes):
9922 adapter = self.db._adapter
9923 tablenames = adapter.tables(self.query,
9924 attributes.get('join',None),
9925 attributes.get('left',None),
9926 attributes.get('orderby',None),
9927 attributes.get('groupby',None))
9928 fields = adapter.expand_all(fields, tablenames)
9929 return adapter._select(self.query,fields,attributes)
9930
9932 db = self.db
9933 tablename = db._adapter.get_table(self.query)
9934 return db._adapter._delete(tablename,self.query)
9935
9936 - def _update(self, **update_fields):
9941
9942 - def as_dict(self, flat=False, sanitize=True):
9943 if flat:
9944 uid = dbname = uri = None
9945 codec = self.db._db_codec
9946 if not sanitize:
9947 uri, dbname, uid = (self.db._dbname, str(self.db),
9948 self.db._db_uid)
9949 d = {"query": self.query.as_dict(flat=flat)}
9950 d["db"] = {"uid": uid, "codec": codec,
9951 "name": dbname, "uri": uri}
9952 return d
9953 else: return self.__dict__
9954
9955 - def as_xml(self, sanitize=True):
9962
9963 - def as_json(self, sanitize=True):
9970
9971 - def parse(self, dquery):
9972 "Experimental: Turn a dictionary into a Query object"
9973 self.dquery = dquery
9974 return self.build(self.dquery)
9975
9977 "Experimental: see .parse()"
9978 op, first, second = (d["op"], d["first"],
9979 d.get("second", None))
9980 left = right = built = None
9981
9982 if op in ("AND", "OR"):
9983 if not (type(first), type(second)) == (dict, dict):
9984 raise SyntaxError("Invalid AND/OR query")
9985 if op == "AND":
9986 built = self.build(first) & self.build(second)
9987 else: built = self.build(first) | self.build(second)
9988
9989 elif op == "NOT":
9990 if first is None:
9991 raise SyntaxError("Invalid NOT query")
9992 built = ~self.build(first)
9993 else:
9994
9995 for k, v in {"left": first, "right": second}.items():
9996 if isinstance(v, dict) and v.get("op"):
9997 v = self.build(v)
9998 if isinstance(v, dict) and ("tablename" in v):
9999 v = self.db[v["tablename"]][v["fieldname"]]
10000 if k == "left": left = v
10001 else: right = v
10002
10003 if hasattr(self.db._adapter, op):
10004 opm = getattr(self.db._adapter, op)
10005
10006 if op == "EQ": built = left == right
10007 elif op == "NE": built = left != right
10008 elif op == "GT": built = left > right
10009 elif op == "GE": built = left >= right
10010 elif op == "LT": built = left < right
10011 elif op == "LE": built = left <= right
10012 elif op in ("JOIN", "LEFT_JOIN", "RANDOM", "ALLOW_NULL"):
10013 built = Expression(self.db, opm)
10014 elif op in ("LOWER", "UPPER", "EPOCH", "PRIMARY_KEY",
10015 "COALESCE_ZERO", "RAW", "INVERT"):
10016 built = Expression(self.db, opm, left)
10017 elif op in ("COUNT", "EXTRACT", "AGGREGATE", "SUBSTRING",
10018 "REGEXP", "LIKE", "ILIKE", "STARTSWITH",
10019 "ENDSWITH", "ADD", "SUB", "MUL", "DIV",
10020 "MOD", "AS", "ON", "COMMA", "NOT_NULL",
10021 "COALESCE", "CONTAINS", "BELONGS"):
10022 built = Expression(self.db, opm, left, right)
10023
10024 elif not (left or right): built = Expression(self.db, op)
10025 else:
10026 raise SyntaxError("Operator not supported: %s" % op)
10027
10028 return built
10029
10031 return not self.select(limitby=(0,1), orderby_on_limitby=False)
10032
10033 - def count(self,distinct=None, cache=None):
10034 db = self.db
10035 if cache:
10036 cache_model, time_expire = cache
10037 sql = self._count(distinct=distinct)
10038 key = db._uri + '/' + sql
10039 if len(key)>200: key = hashlib_md5(key).hexdigest()
10040 return cache_model(
10041 key,
10042 (lambda self=self,distinct=distinct: \
10043 db._adapter.count(self.query,distinct)),
10044 time_expire)
10045 return db._adapter.count(self.query,distinct)
10046
10047 - def select(self, *fields, **attributes):
10048 adapter = self.db._adapter
10049 tablenames = adapter.tables(self.query,
10050 attributes.get('join',None),
10051 attributes.get('left',None),
10052 attributes.get('orderby',None),
10053 attributes.get('groupby',None))
10054 fields = adapter.expand_all(fields, tablenames)
10055 return adapter.select(self.query,fields,attributes)
10056
10059
10061 db = self.db
10062 tablename = db._adapter.get_table(self.query)
10063 table = db[tablename]
10064 if any(f(self) for f in table._before_delete): return 0
10065 ret = db._adapter.delete(tablename,self.query)
10066 ret and [f(self) for f in table._after_delete]
10067 return ret
10068
10069 - def update(self, **update_fields):
10070 db = self.db
10071 tablename = db._adapter.get_table(self.query)
10072 table = db[tablename]
10073 table._attempt_upload(update_fields)
10074 if any(f(self,update_fields) for f in table._before_update):
10075 return 0
10076 fields = table._listify(update_fields,update=True)
10077 if not fields:
10078 raise SyntaxError("No fields to update")
10079 ret = db._adapter.update("%s" % table,self.query,fields)
10080 ret and [f(self,update_fields) for f in table._after_update]
10081 return ret
10082
10084 """
10085 same as update but does not call table._before_update and _after_update
10086 """
10087 tablename = self.db._adapter.get_table(self.query)
10088 table = self.db[tablename]
10089 fields = table._listify(update_fields,update=True)
10090 if not fields: raise SyntaxError("No fields to update")
10091
10092 ret = self.db._adapter.update("%s" % table,self.query,fields)
10093 return ret
10094
10096 tablename = self.db._adapter.get_table(self.query)
10097 response = Row()
10098 response.errors = Row()
10099 new_fields = copy.copy(update_fields)
10100 for key,value in update_fields.iteritems():
10101 value,error = self.db[tablename][key].validate(value)
10102 if error:
10103 response.errors[key] = error
10104 else:
10105 new_fields[key] = value
10106 table = self.db[tablename]
10107 if response.errors:
10108 response.updated = None
10109 else:
10110 if not any(f(self,new_fields) for f in table._before_update):
10111 fields = table._listify(new_fields,update=True)
10112 if not fields: raise SyntaxError("No fields to update")
10113 ret = self.db._adapter.update(tablename,self.query,fields)
10114 ret and [f(self,new_fields) for f in table._after_update]
10115 else:
10116 ret = 0
10117 response.updated = ret
10118 return response
10119
10121 table = self.db[self.db._adapter.tables(self.query)[0]]
10122
10123 if upload_fields:
10124 fields = upload_fields.keys()
10125 else:
10126 fields = table.fields
10127 fields = [f for f in fields if table[f].type == 'upload'
10128 and table[f].uploadfield == True
10129 and table[f].autodelete]
10130 if not fields:
10131 return False
10132 for record in self.select(*[table[f] for f in fields]):
10133 for fieldname in fields:
10134 field = table[fieldname]
10135 oldname = record.get(fieldname, None)
10136 if not oldname:
10137 continue
10138 if upload_fields and oldname == upload_fields[fieldname]:
10139 continue
10140 if field.custom_delete:
10141 field.custom_delete(oldname)
10142 else:
10143 uploadfolder = field.uploadfolder
10144 if not uploadfolder:
10145 uploadfolder = pjoin(
10146 self.db._adapter.folder, '..', 'uploads')
10147 if field.uploadseparate:
10148 items = oldname.split('.')
10149 uploadfolder = pjoin(
10150 uploadfolder,
10151 "%s.%s" % (items[0], items[1]),
10152 items[2][:2])
10153 oldpath = pjoin(uploadfolder, oldname)
10154 if exists(oldpath):
10155 os.unlink(oldpath)
10156 return False
10157
10159 - def __init__(self, colset, table, id):
10160 self.colset, self.db, self.tablename, self.id = \
10161 colset, table._db, table._tablename, id
10162
10164 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
10165 table = db[tablename]
10166 newfields = fields or dict(colset)
10167 for fieldname in newfields.keys():
10168 if not fieldname in table.fields or table[fieldname].type=='id':
10169 del newfields[fieldname]
10170 table._db(table._id==id,ignore_common_filters=True).update(**newfields)
10171 colset.update(newfields)
10172 return colset
10173
10176 self.db, self.tablename, self.id = table._db, table._tablename, id
10178 return self.db(self.db[self.tablename]._id==self.id).delete()
10179
10182 self.db, self.tablename, self.id = table._db, table._tablename, id
10184 if self.db._lazy_tables is False:
10185 raise AttributeError()
10186 table = self.db[self.tablename]
10187 other_table = self.db[other_tablename]
10188 for rfield in table._referenced_by:
10189 if rfield.table == other_table:
10190 return LazySet(rfield, self.id)
10191
10192 raise AttributeError()
10193
10196 self.db, self.tablename, self.fieldname, self.id = \
10197 field.db, field._tablename, field.name, id
10199 query = self.db[self.tablename][self.fieldname]==self.id
10200 return Set(self.db,query)
10203 - def __call__(self, query, ignore_common_filters=False):
10204 return self._getset()(query, ignore_common_filters)
10205 - def _count(self,distinct=None):
10207 - def _select(self, *fields, **attributes):
10211 - def _update(self, **update_fields):
10215 - def count(self,distinct=None, cache=None):
10217 - def select(self, *fields, **attributes):
10223 - def update(self, **update_fields):
10231
10234 self.method=method
10235 self.row=row
10237 return self.method(self.row,*args,**kwargs)
10238
10240 f.__lazy__ = True
10241 return f
10242
10243 -class Rows(object):
10244
10245 """
10246 A wrapper for the return value of a select. It basically represents a table.
10247 It has an iterator and each row is represented as a dictionary.
10248 """
10249
10250
10251
10252 - def __init__(
10253 self,
10254 db=None,
10255 records=[],
10256 colnames=[],
10257 compact=True,
10258 rawrows=None
10259 ):
10260 self.db = db
10261 self.records = records
10262 self.colnames = colnames
10263 self.compact = compact
10264 self.response = rawrows
10265
10267 return '<Rows (%s)>' % len(self.records)
10268
10270 """
10271 db.define_table('x',Field('number','integer'))
10272 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
10273
10274 from gluon.dal import lazy_virtualfield
10275
10276 class MyVirtualFields(object):
10277 # normal virtual field (backward compatible, discouraged)
10278 def normal_shift(self): return self.x.number+1
10279 # lazy virtual field (because of @staticmethod)
10280 @lazy_virtualfield
10281 def lazy_shift(instance,row,delta=4): return row.x.number+delta
10282 db.x.virtualfields.append(MyVirtualFields())
10283
10284 for row in db(db.x).select():
10285 print row.number, row.normal_shift, row.lazy_shift(delta=7)
10286 """
10287 if not keyed_virtualfields:
10288 return self
10289 for row in self.records:
10290 for (tablename,virtualfields) in keyed_virtualfields.iteritems():
10291 attributes = dir(virtualfields)
10292 if not tablename in row:
10293 box = row[tablename] = Row()
10294 else:
10295 box = row[tablename]
10296 updated = False
10297 for attribute in attributes:
10298 if attribute[0] != '_':
10299 method = getattr(virtualfields,attribute)
10300 if hasattr(method,'__lazy__'):
10301 box[attribute]=VirtualCommand(method,row)
10302 elif type(method)==types.MethodType:
10303 if not updated:
10304 virtualfields.__dict__.update(row)
10305 updated = True
10306 box[attribute]=method()
10307 return self
10308
10310 if self.colnames!=other.colnames:
10311 raise Exception('Cannot & incompatible Rows objects')
10312 records = self.records+other.records
10313 return Rows(self.db,records,self.colnames)
10314
10316 if self.colnames!=other.colnames:
10317 raise Exception('Cannot | incompatible Rows objects')
10318 records = self.records
10319 records += [record for record in other.records \
10320 if not record in records]
10321 return Rows(self.db,records,self.colnames)
10322
10324 if len(self.records):
10325 return 1
10326 return 0
10327
10329 return len(self.records)
10330
10332 return Rows(self.db,self.records[a:b],self.colnames,compact=self.compact)
10333
10335 row = self.records[i]
10336 keys = row.keys()
10337 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10338 return row[row.keys()[0]]
10339 return row
10340
10342 """
10343 iterator over records
10344 """
10345
10346 for i in xrange(len(self)):
10347 yield self[i]
10348
10350 """
10351 serializes the table into a csv file
10352 """
10353
10354 s = StringIO.StringIO()
10355 self.export_to_csv_file(s)
10356 return s.getvalue()
10357
10359 if not self.records:
10360 return None
10361 return self[0]
10362
10364 if not self.records:
10365 return None
10366 return self[-1]
10367
10368 - def find(self,f,limitby=None):
10369 """
10370 returns a new Rows object, a subset of the original object,
10371 filtered by the function f
10372 """
10373 if not self:
10374 return Rows(self.db, [], self.colnames)
10375 records = []
10376 if limitby:
10377 a,b = limitby
10378 else:
10379 a,b = 0,len(self)
10380 k = 0
10381 for row in self:
10382 if f(row):
10383 if a<=k: records.append(row)
10384 k += 1
10385 if k==b: break
10386 return Rows(self.db, records, self.colnames)
10387
10389 """
10390 removes elements from the calling Rows object, filtered by the function f,
10391 and returns a new Rows object containing the removed elements
10392 """
10393 if not self.records:
10394 return Rows(self.db, [], self.colnames)
10395 removed = []
10396 i=0
10397 while i<len(self):
10398 row = self[i]
10399 if f(row):
10400 removed.append(self.records[i])
10401 del self.records[i]
10402 else:
10403 i += 1
10404 return Rows(self.db, removed, self.colnames)
10405
10406 - def sort(self, f, reverse=False):
10407 """
10408 returns a list of sorted elements (not sorted in place)
10409 """
10410 rows = Rows(self.db,[],self.colnames,compact=False)
10411 rows.records = sorted(self,key=f,reverse=reverse)
10412 return rows
10413
10415 """
10416 regroups the rows, by one of the fields
10417 """
10418 one_result = False
10419 if 'one_result' in args:
10420 one_result = args['one_result']
10421
10422 def build_fields_struct(row, fields, num, groups):
10423 ''' helper function:
10424 '''
10425 if num > len(fields)-1:
10426 if one_result:
10427 return row
10428 else:
10429 return [row]
10430
10431 key = fields[num]
10432 value = row[key]
10433
10434 if value not in groups:
10435 groups[value] = build_fields_struct(row, fields, num+1, {})
10436 else:
10437 struct = build_fields_struct(row, fields, num+1, groups[ value ])
10438
10439
10440 if type(struct) == type(dict()):
10441 groups[value].update()
10442
10443 elif type(struct) == type(list()):
10444 groups[value] += struct
10445
10446 else:
10447 groups[value] = struct
10448
10449 return groups
10450
10451 if len(fields) == 0:
10452 return self
10453
10454
10455 if not self.records:
10456 return {}
10457
10458 grouped_row_group = dict()
10459
10460
10461 for row in self:
10462 build_fields_struct(row, fields, 0, grouped_row_group)
10463
10464 return grouped_row_group
10465
10466 - def render(self, i=None, fields=None):
10467 """
10468 Takes an index and returns a copy of the indexed row with values
10469 transformed via the "represent" attributes of the associated fields.
10470
10471 If no index is specified, a generator is returned for iteration
10472 over all the rows.
10473
10474 fields -- a list of fields to transform (if None, all fields with
10475 "represent" attributes will be transformed).
10476 """
10477
10478
10479 if i is None:
10480 return (self.repr(i, fields=fields) for i in range(len(self)))
10481 import sqlhtml
10482 row = copy.deepcopy(self.records[i])
10483 keys = row.keys()
10484 tables = [f.tablename for f in fields] if fields \
10485 else [k for k in keys if k != '_extra']
10486 for table in tables:
10487 repr_fields = [f.name for f in fields if f.tablename == table] \
10488 if fields else [k for k in row[table].keys()
10489 if (hasattr(self.db[table], k) and
10490 isinstance(self.db[table][k], Field)
10491 and self.db[table][k].represent)]
10492 for field in repr_fields:
10493 row[table][field] = sqlhtml.represent(
10494 self.db[table][field], row[table][field], row[table])
10495 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10496 return row[keys[0]]
10497 return row
10498
10499 - def as_list(self,
10500 compact=True,
10501 storage_to_dict=True,
10502 datetime_to_str=False,
10503 custom_types=None):
10504 """
10505 returns the data as a list or dictionary.
10506 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10507 :param datetime_to_str: convert datetime fields as strings (default False)
10508 """
10509 (oc, self.compact) = (self.compact, compact)
10510 if storage_to_dict:
10511 items = [item.as_dict(datetime_to_str, custom_types) for item in self]
10512 else:
10513 items = [item for item in self]
10514 self.compact = compact
10515 return items
10516
10517
10518 - def as_dict(self,
10519 key='id',
10520 compact=True,
10521 storage_to_dict=True,
10522 datetime_to_str=False,
10523 custom_types=None):
10524 """
10525 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
10526
10527 :param key: the name of the field to be used as dict key, normally the id
10528 :param compact: ? (default True)
10529 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10530 :param datetime_to_str: convert datetime fields as strings (default False)
10531 """
10532
10533
10534 multi = False
10535 f = self.first()
10536 if f and isinstance(key, basestring):
10537 multi = any([isinstance(v, f.__class__) for v in f.values()])
10538 if (not "." in key) and multi:
10539
10540 def new_key():
10541 i = 0
10542 while True:
10543 yield i
10544 i += 1
10545 key_generator = new_key()
10546 key = lambda r: key_generator.next()
10547
10548 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types)
10549 if isinstance(key,str) and key.count('.')==1:
10550 (table, field) = key.split('.')
10551 return dict([(r[table][field],r) for r in rows])
10552 elif isinstance(key,str):
10553 return dict([(r[key],r) for r in rows])
10554 else:
10555 return dict([(key(r),r) for r in rows])
10556
10558 """
10559 export data to csv, the first line contains the column names
10560
10561 :param ofile: where the csv must be exported to
10562 :param null: how null values must be represented (default '<NULL>')
10563 :param delimiter: delimiter to separate values (default ',')
10564 :param quotechar: character to use to quote string values (default '"')
10565 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
10566 :param represent: use the fields .represent value (default False)
10567 :param colnames: list of column names to use (default self.colnames)
10568 This will only work when exporting rows objects!!!!
10569 DO NOT use this with db.export_to_csv()
10570 """
10571 delimiter = kwargs.get('delimiter', ',')
10572 quotechar = kwargs.get('quotechar', '"')
10573 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
10574 represent = kwargs.get('represent', False)
10575 writer = csv.writer(ofile, delimiter=delimiter,
10576 quotechar=quotechar, quoting=quoting)
10577 colnames = kwargs.get('colnames', self.colnames)
10578 write_colnames = kwargs.get('write_colnames',True)
10579
10580 if write_colnames:
10581 writer.writerow(colnames)
10582
10583 def none_exception(value):
10584 """
10585 returns a cleaned up value that can be used for csv export:
10586 - unicode text is encoded as such
10587 - None values are replaced with the given representation (default <NULL>)
10588 """
10589 if value is None:
10590 return null
10591 elif isinstance(value, unicode):
10592 return value.encode('utf8')
10593 elif isinstance(value,Reference):
10594 return long(value)
10595 elif hasattr(value, 'isoformat'):
10596 return value.isoformat()[:19].replace('T', ' ')
10597 elif isinstance(value, (list,tuple)):
10598 return bar_encode(value)
10599 return value
10600
10601 for record in self:
10602 row = []
10603 for col in colnames:
10604 if not REGEX_TABLE_DOT_FIELD.match(col):
10605 row.append(record._extra[col])
10606 else:
10607 (t, f) = col.split('.')
10608 field = self.db[t][f]
10609 if isinstance(record.get(t, None), (Row,dict)):
10610 value = record[t][f]
10611 else:
10612 value = record[f]
10613 if field.type=='blob' and not value is None:
10614 value = base64.b64encode(value)
10615 elif represent and field.represent:
10616 value = field.represent(value)
10617 row.append(none_exception(value))
10618 writer.writerow(row)
10619
10620 - def xml(self,strict=False,row_name='row',rows_name='rows'):
10621 """
10622 serializes the table using sqlhtml.SQLTABLE (if present)
10623 """
10624
10625 if strict:
10626 ncols = len(self.colnames)
10627 return '<%s>\n%s\n</%s>' % (rows_name,
10628 '\n'.join(row.as_xml(row_name=row_name,
10629 colnames=self.colnames) for
10630 row in self), rows_name)
10631
10632 import sqlhtml
10633 return sqlhtml.SQLTABLE(self).xml()
10634
10635 - def as_xml(self,row_name='row',rows_name='rows'):
10636 return self.xml(strict=True, row_name=row_name, rows_name=rows_name)
10637
10638 - def as_json(self, mode='object', default=None):
10639 """
10640 serializes the rows to a JSON list or object with objects
10641 mode='object' is not implemented (should return a nested
10642 object structure)
10643 """
10644
10645 items = [record.as_json(mode=mode, default=default,
10646 serialize=False,
10647 colnames=self.colnames) for
10648 record in self]
10649
10650 if have_serializers:
10651 return serializers.json(items,
10652 default=default or
10653 serializers.custom_json)
10654 elif simplejson:
10655 return simplejson.dumps(items)
10656 else:
10657 raise RuntimeError("missing simplejson")
10658
10659
10660 as_csv = __str__
10661 json = as_json
10662
10663
10664
10665
10666
10667
10668 -def test_all():
10669 """
10670
10671 >>> if len(sys.argv)<2: db = DAL("sqlite://test.db")
10672 >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
10673 >>> tmp = db.define_table('users',\
10674 Field('stringf', 'string', length=32, required=True),\
10675 Field('booleanf', 'boolean', default=False),\
10676 Field('passwordf', 'password', notnull=True),\
10677 Field('uploadf', 'upload'),\
10678 Field('blobf', 'blob'),\
10679 Field('integerf', 'integer', unique=True),\
10680 Field('doublef', 'double', unique=True,notnull=True),\
10681 Field('jsonf', 'json'),\
10682 Field('datef', 'date', default=datetime.date.today()),\
10683 Field('timef', 'time'),\
10684 Field('datetimef', 'datetime'),\
10685 migrate='test_user.table')
10686
10687 Insert a field
10688
10689 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
10690 uploadf=None, integerf=5, doublef=3.14,\
10691 jsonf={"j": True},\
10692 datef=datetime.date(2001, 1, 1),\
10693 timef=datetime.time(12, 30, 15),\
10694 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
10695 1
10696
10697 Drop the table
10698
10699 >>> db.users.drop()
10700
10701 Examples of insert, select, update, delete
10702
10703 >>> tmp = db.define_table('person',\
10704 Field('name'),\
10705 Field('birth','date'),\
10706 migrate='test_person.table')
10707 >>> person_id = db.person.insert(name='Marco',birth='2005-06-22')
10708 >>> person_id = db.person.insert(name='Massimo',birth='1971-12-21')
10709
10710 commented len(db().select(db.person.ALL))
10711 commented 2
10712
10713 >>> me = db(db.person.id==person_id).select()[0] # test select
10714 >>> me.name
10715 'Massimo'
10716 >>> db.person[2].name
10717 'Massimo'
10718 >>> db.person(2).name
10719 'Massimo'
10720 >>> db.person(name='Massimo').name
10721 'Massimo'
10722 >>> db.person(db.person.name=='Massimo').name
10723 'Massimo'
10724 >>> row = db.person[2]
10725 >>> row.name == row['name'] == row['person.name'] == row('person.name')
10726 True
10727 >>> db(db.person.name=='Massimo').update(name='massimo') # test update
10728 1
10729 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
10730 1
10731
10732 Update a single record
10733
10734 >>> me.update_record(name="Max")
10735 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}>
10736 >>> me.name
10737 'Max'
10738
10739 Examples of complex search conditions
10740
10741 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
10742 1
10743 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
10744 1
10745 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
10746 1
10747 >>> me = db(db.person.id==person_id).select(db.person.name)[0]
10748 >>> me.name
10749 'Max'
10750
10751 Examples of search conditions using extract from date/datetime/time
10752
10753 >>> len(db(db.person.birth.month()==12).select())
10754 1
10755 >>> len(db(db.person.birth.year()>1900).select())
10756 1
10757
10758 Example of usage of NULL
10759
10760 >>> len(db(db.person.birth==None).select()) ### test NULL
10761 0
10762 >>> len(db(db.person.birth!=None).select()) ### test NULL
10763 1
10764
10765 Examples of search conditions using lower, upper, and like
10766
10767 >>> len(db(db.person.name.upper()=='MAX').select())
10768 1
10769 >>> len(db(db.person.name.like('%ax')).select())
10770 1
10771 >>> len(db(db.person.name.upper().like('%AX')).select())
10772 1
10773 >>> len(db(~db.person.name.upper().like('%AX')).select())
10774 0
10775
10776 orderby, groupby and limitby
10777
10778 >>> people = db().select(db.person.name, orderby=db.person.name)
10779 >>> order = db.person.name|~db.person.birth
10780 >>> people = db().select(db.person.name, orderby=order)
10781
10782 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
10783
10784 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
10785
10786 Example of one 2 many relation
10787
10788 >>> tmp = db.define_table('dog',\
10789 Field('name'),\
10790 Field('birth','date'),\
10791 Field('owner',db.person),\
10792 migrate='test_dog.table')
10793 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
10794 1
10795
10796 A simple JOIN
10797
10798 >>> len(db(db.dog.owner==db.person.id).select())
10799 1
10800
10801 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
10802 1
10803
10804 Drop tables
10805
10806 >>> db.dog.drop()
10807 >>> db.person.drop()
10808
10809 Example of many 2 many relation and Set
10810
10811 >>> tmp = db.define_table('author', Field('name'),\
10812 migrate='test_author.table')
10813 >>> tmp = db.define_table('paper', Field('title'),\
10814 migrate='test_paper.table')
10815 >>> tmp = db.define_table('authorship',\
10816 Field('author_id', db.author),\
10817 Field('paper_id', db.paper),\
10818 migrate='test_authorship.table')
10819 >>> aid = db.author.insert(name='Massimo')
10820 >>> pid = db.paper.insert(title='QCD')
10821 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
10822
10823 Define a Set
10824
10825 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
10826 >>> rows = authored_papers.select(db.author.name, db.paper.title)
10827 >>> for row in rows: print row.author.name, row.paper.title
10828 Massimo QCD
10829
10830 Example of search condition using belongs
10831
10832 >>> set = (1, 2, 3)
10833 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
10834 >>> print rows[0].title
10835 QCD
10836
10837 Example of search condition using nested select
10838
10839 >>> nested_select = db()._select(db.authorship.paper_id)
10840 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
10841 >>> print rows[0].title
10842 QCD
10843
10844 Example of expressions
10845
10846 >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
10847 >>> db(mynumber).delete()
10848 0
10849 >>> for i in range(10): tmp = mynumber.insert(x=i)
10850 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum())
10851 45
10852
10853 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
10854 5
10855
10856 Output in csv
10857
10858 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
10859 author.name,paper.title\r
10860 Massimo,QCD
10861
10862 Delete all leftover tables
10863
10864 >>> DAL.distributed_transaction_commit(db)
10865
10866 >>> db.mynumber.drop()
10867 >>> db.authorship.drop()
10868 >>> db.author.drop()
10869 >>> db.paper.drop()
10870 """
10871
10872
10873
10874
10875 SQLField = Field
10876 SQLTable = Table
10877 SQLXorable = Expression
10878 SQLQuery = Query
10879 SQLSet = Set
10880 SQLRows = Rows
10881 SQLStorage = Row
10882 SQLDB = DAL
10883 GQLDB = DAL
10884 DAL.Field = Field
10885 DAL.Table = Table
10886
10887
10888
10889
10890
10891 -def geoPoint(x,y):
10892 return "POINT (%f %f)" % (x,y)
10893
10895 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
10896
10898 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
10899
10900
10901
10902
10903
10904 if __name__ == '__main__':
10905 import doctest
10906 doctest.testmod()
10907